*.pyc
docker/*/generated/*
docker/config.yml
-doc/_site/*
doc/.site/*
-doc/sdk/python/arvados
\ No newline at end of file
+doc/sdk/python/arvados
+sdk/perl/MYMETA.*
+sdk/perl/Makefile
+sdk/perl/blib/*
+sdk/perl/pm_to_blib
--- /dev/null
+Server-side components of Arvados contained in the apps/ and services/
+directories, including the API Server, Workbench, and Crunch, are licenced
+under the GNU Affero General Public License version 3 (see agpl-3.0.txt)
+
+The Arvados client Software Development Kits contained in the sdk/ directory,
+example scripts in the crunch_scripts/ directory, and code samples in the
+Aravados documentation are licensed under the Apache License, Version 2.0 (see
+LICENSE-2.0.txt)
+
+The Arvados Documentation located in the doc/ directory is licensed under the
+Creative Commons Attribution-Share Alike 3.0 United States (see by-sa-3.0.txt)
\ No newline at end of file
--- /dev/null
+
+ Apache License
+ Version 2.0, January 2004
+ http://www.apache.org/licenses/
+
+ TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
+
+ 1. Definitions.
+
+ "License" shall mean the terms and conditions for use, reproduction,
+ and distribution as defined by Sections 1 through 9 of this document.
+
+ "Licensor" shall mean the copyright owner or entity authorized by
+ the copyright owner that is granting the License.
+
+ "Legal Entity" shall mean the union of the acting entity and all
+ other entities that control, are controlled by, or are under common
+ control with that entity. For the purposes of this definition,
+ "control" means (i) the power, direct or indirect, to cause the
+ direction or management of such entity, whether by contract or
+ otherwise, or (ii) ownership of fifty percent (50%) or more of the
+ outstanding shares, or (iii) beneficial ownership of such entity.
+
+ "You" (or "Your") shall mean an individual or Legal Entity
+ exercising permissions granted by this License.
+
+ "Source" form shall mean the preferred form for making modifications,
+ including but not limited to software source code, documentation
+ source, and configuration files.
+
+ "Object" form shall mean any form resulting from mechanical
+ transformation or translation of a Source form, including but
+ not limited to compiled object code, generated documentation,
+ and conversions to other media types.
+
+ "Work" shall mean the work of authorship, whether in Source or
+ Object form, made available under the License, as indicated by a
+ copyright notice that is included in or attached to the work
+ (an example is provided in the Appendix below).
+
+ "Derivative Works" shall mean any work, whether in Source or Object
+ form, that is based on (or derived from) the Work and for which the
+ editorial revisions, annotations, elaborations, or other modifications
+ represent, as a whole, an original work of authorship. For the purposes
+ of this License, Derivative Works shall not include works that remain
+ separable from, or merely link (or bind by name) to the interfaces of,
+ the Work and Derivative Works thereof.
+
+ "Contribution" shall mean any work of authorship, including
+ the original version of the Work and any modifications or additions
+ to that Work or Derivative Works thereof, that is intentionally
+ submitted to Licensor for inclusion in the Work by the copyright owner
+ or by an individual or Legal Entity authorized to submit on behalf of
+ the copyright owner. For the purposes of this definition, "submitted"
+ means any form of electronic, verbal, or written communication sent
+ to the Licensor or its representatives, including but not limited to
+ communication on electronic mailing lists, source code control systems,
+ and issue tracking systems that are managed by, or on behalf of, the
+ Licensor for the purpose of discussing and improving the Work, but
+ excluding communication that is conspicuously marked or otherwise
+ designated in writing by the copyright owner as "Not a Contribution."
+
+ "Contributor" shall mean Licensor and any individual or Legal Entity
+ on behalf of whom a Contribution has been received by Licensor and
+ subsequently incorporated within the Work.
+
+ 2. Grant of Copyright License. Subject to the terms and conditions of
+ this License, each Contributor hereby grants to You a perpetual,
+ worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+ copyright license to reproduce, prepare Derivative Works of,
+ publicly display, publicly perform, sublicense, and distribute the
+ Work and such Derivative Works in Source or Object form.
+
+ 3. Grant of Patent License. Subject to the terms and conditions of
+ this License, each Contributor hereby grants to You a perpetual,
+ worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+ (except as stated in this section) patent license to make, have made,
+ use, offer to sell, sell, import, and otherwise transfer the Work,
+ where such license applies only to those patent claims licensable
+ by such Contributor that are necessarily infringed by their
+ Contribution(s) alone or by combination of their Contribution(s)
+ with the Work to which such Contribution(s) was submitted. If You
+ institute patent litigation against any entity (including a
+ cross-claim or counterclaim in a lawsuit) alleging that the Work
+ or a Contribution incorporated within the Work constitutes direct
+ or contributory patent infringement, then any patent licenses
+ granted to You under this License for that Work shall terminate
+ as of the date such litigation is filed.
+
+ 4. Redistribution. You may reproduce and distribute copies of the
+ Work or Derivative Works thereof in any medium, with or without
+ modifications, and in Source or Object form, provided that You
+ meet the following conditions:
+
+ (a) You must give any other recipients of the Work or
+ Derivative Works a copy of this License; and
+
+ (b) You must cause any modified files to carry prominent notices
+ stating that You changed the files; and
+
+ (c) You must retain, in the Source form of any Derivative Works
+ that You distribute, all copyright, patent, trademark, and
+ attribution notices from the Source form of the Work,
+ excluding those notices that do not pertain to any part of
+ the Derivative Works; and
+
+ (d) If the Work includes a "NOTICE" text file as part of its
+ distribution, then any Derivative Works that You distribute must
+ include a readable copy of the attribution notices contained
+ within such NOTICE file, excluding those notices that do not
+ pertain to any part of the Derivative Works, in at least one
+ of the following places: within a NOTICE text file distributed
+ as part of the Derivative Works; within the Source form or
+ documentation, if provided along with the Derivative Works; or,
+ within a display generated by the Derivative Works, if and
+ wherever such third-party notices normally appear. The contents
+ of the NOTICE file are for informational purposes only and
+ do not modify the License. You may add Your own attribution
+ notices within Derivative Works that You distribute, alongside
+ or as an addendum to the NOTICE text from the Work, provided
+ that such additional attribution notices cannot be construed
+ as modifying the License.
+
+ You may add Your own copyright statement to Your modifications and
+ may provide additional or different license terms and conditions
+ for use, reproduction, or distribution of Your modifications, or
+ for any such Derivative Works as a whole, provided Your use,
+ reproduction, and distribution of the Work otherwise complies with
+ the conditions stated in this License.
+
+ 5. Submission of Contributions. Unless You explicitly state otherwise,
+ any Contribution intentionally submitted for inclusion in the Work
+ by You to the Licensor shall be under the terms and conditions of
+ this License, without any additional terms or conditions.
+ Notwithstanding the above, nothing herein shall supersede or modify
+ the terms of any separate license agreement you may have executed
+ with Licensor regarding such Contributions.
+
+ 6. Trademarks. This License does not grant permission to use the trade
+ names, trademarks, service marks, or product names of the Licensor,
+ except as required for reasonable and customary use in describing the
+ origin of the Work and reproducing the content of the NOTICE file.
+
+ 7. Disclaimer of Warranty. Unless required by applicable law or
+ agreed to in writing, Licensor provides the Work (and each
+ Contributor provides its Contributions) on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+ implied, including, without limitation, any warranties or conditions
+ of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
+ PARTICULAR PURPOSE. You are solely responsible for determining the
+ appropriateness of using or redistributing the Work and assume any
+ risks associated with Your exercise of permissions under this License.
+
+ 8. Limitation of Liability. In no event and under no legal theory,
+ whether in tort (including negligence), contract, or otherwise,
+ unless required by applicable law (such as deliberate and grossly
+ negligent acts) or agreed to in writing, shall any Contributor be
+ liable to You for damages, including any direct, indirect, special,
+ incidental, or consequential damages of any character arising as a
+ result of this License or out of the use or inability to use the
+ Work (including but not limited to damages for loss of goodwill,
+ work stoppage, computer failure or malfunction, or any and all
+ other commercial damages or losses), even if such Contributor
+ has been advised of the possibility of such damages.
+
+ 9. Accepting Warranty or Additional Liability. While redistributing
+ the Work or Derivative Works thereof, You may choose to offer,
+ and charge a fee for, acceptance of support, warranty, indemnity,
+ or other liability obligations and/or rights consistent with this
+ License. However, in accepting such obligations, You may act only
+ on Your own behalf and on Your sole responsibility, not on behalf
+ of any other Contributor, and only if You agree to indemnify,
+ defend, and hold each Contributor harmless for any liability
+ incurred by, or claims asserted against, such Contributor by reason
+ of your accepting any such warranty or additional liability.
+
+ END OF TERMS AND CONDITIONS
+
+ APPENDIX: How to apply the Apache License to your work.
+
+ To apply the Apache License to your work, attach the following
+ boilerplate notice, with the fields enclosed by brackets "[]"
+ replaced with your own identifying information. (Don't include
+ the brackets!) The text should be enclosed in the appropriate
+ comment syntax for the file format. We also recommend that a
+ file or class name and description of purpose be included on the
+ same "printed page" as the copyright notice for easier
+ identification within third-party archives.
+
+ Copyright [yyyy] [name of copyright owner]
+
+ Licensed under the Apache License, Version 2.0 (the "License");
+ you may not use this file except in compliance with the License.
+ You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
--- /dev/null
+Welcome to Arvados!
+
+The main Arvados web site is
+ https://arvados.org
+
+The Arvados public wiki is located at
+ https://arvados.org/projects/arvados/wiki
+
+The Arvados public bug tracker is located at
+ https://arvados.org/projects/arvados/issues
+
+For support see
+ http://doc.arvados.org/user/getting_started/community.html
+
+Installation documentation is located at
+ http://doc.arvados.org/install
+
+If you wish to build the documentation yourself, follow the instructions in
+doc/README to build the documentation, then consult the "Install Guide".
+
+See COPYING for information about Arvados Free Software licenses.
--- /dev/null
+ GNU AFFERO GENERAL PUBLIC LICENSE
+ Version 3, 19 November 2007
+
+ Copyright (C) 2007 Free Software Foundation, Inc. <http://fsf.org/>
+ Everyone is permitted to copy and distribute verbatim copies
+ of this license document, but changing it is not allowed.
+
+ Preamble
+
+ The GNU Affero General Public License is a free, copyleft license for
+software and other kinds of works, specifically designed to ensure
+cooperation with the community in the case of network server software.
+
+ The licenses for most software and other practical works are designed
+to take away your freedom to share and change the works. By contrast,
+our General Public Licenses are intended to guarantee your freedom to
+share and change all versions of a program--to make sure it remains free
+software for all its users.
+
+ When we speak of free software, we are referring to freedom, not
+price. Our General Public Licenses are designed to make sure that you
+have the freedom to distribute copies of free software (and charge for
+them if you wish), that you receive source code or can get it if you
+want it, that you can change the software or use pieces of it in new
+free programs, and that you know you can do these things.
+
+ Developers that use our General Public Licenses protect your rights
+with two steps: (1) assert copyright on the software, and (2) offer
+you this License which gives you legal permission to copy, distribute
+and/or modify the software.
+
+ A secondary benefit of defending all users' freedom is that
+improvements made in alternate versions of the program, if they
+receive widespread use, become available for other developers to
+incorporate. Many developers of free software are heartened and
+encouraged by the resulting cooperation. However, in the case of
+software used on network servers, this result may fail to come about.
+The GNU General Public License permits making a modified version and
+letting the public access it on a server without ever releasing its
+source code to the public.
+
+ The GNU Affero General Public License is designed specifically to
+ensure that, in such cases, the modified source code becomes available
+to the community. It requires the operator of a network server to
+provide the source code of the modified version running there to the
+users of that server. Therefore, public use of a modified version, on
+a publicly accessible server, gives the public access to the source
+code of the modified version.
+
+ An older license, called the Affero General Public License and
+published by Affero, was designed to accomplish similar goals. This is
+a different license, not a version of the Affero GPL, but Affero has
+released a new version of the Affero GPL which permits relicensing under
+this license.
+
+ The precise terms and conditions for copying, distribution and
+modification follow.
+
+ TERMS AND CONDITIONS
+
+ 0. Definitions.
+
+ "This License" refers to version 3 of the GNU Affero General Public License.
+
+ "Copyright" also means copyright-like laws that apply to other kinds of
+works, such as semiconductor masks.
+
+ "The Program" refers to any copyrightable work licensed under this
+License. Each licensee is addressed as "you". "Licensees" and
+"recipients" may be individuals or organizations.
+
+ To "modify" a work means to copy from or adapt all or part of the work
+in a fashion requiring copyright permission, other than the making of an
+exact copy. The resulting work is called a "modified version" of the
+earlier work or a work "based on" the earlier work.
+
+ A "covered work" means either the unmodified Program or a work based
+on the Program.
+
+ To "propagate" a work means to do anything with it that, without
+permission, would make you directly or secondarily liable for
+infringement under applicable copyright law, except executing it on a
+computer or modifying a private copy. Propagation includes copying,
+distribution (with or without modification), making available to the
+public, and in some countries other activities as well.
+
+ To "convey" a work means any kind of propagation that enables other
+parties to make or receive copies. Mere interaction with a user through
+a computer network, with no transfer of a copy, is not conveying.
+
+ An interactive user interface displays "Appropriate Legal Notices"
+to the extent that it includes a convenient and prominently visible
+feature that (1) displays an appropriate copyright notice, and (2)
+tells the user that there is no warranty for the work (except to the
+extent that warranties are provided), that licensees may convey the
+work under this License, and how to view a copy of this License. If
+the interface presents a list of user commands or options, such as a
+menu, a prominent item in the list meets this criterion.
+
+ 1. Source Code.
+
+ The "source code" for a work means the preferred form of the work
+for making modifications to it. "Object code" means any non-source
+form of a work.
+
+ A "Standard Interface" means an interface that either is an official
+standard defined by a recognized standards body, or, in the case of
+interfaces specified for a particular programming language, one that
+is widely used among developers working in that language.
+
+ The "System Libraries" of an executable work include anything, other
+than the work as a whole, that (a) is included in the normal form of
+packaging a Major Component, but which is not part of that Major
+Component, and (b) serves only to enable use of the work with that
+Major Component, or to implement a Standard Interface for which an
+implementation is available to the public in source code form. A
+"Major Component", in this context, means a major essential component
+(kernel, window system, and so on) of the specific operating system
+(if any) on which the executable work runs, or a compiler used to
+produce the work, or an object code interpreter used to run it.
+
+ The "Corresponding Source" for a work in object code form means all
+the source code needed to generate, install, and (for an executable
+work) run the object code and to modify the work, including scripts to
+control those activities. However, it does not include the work's
+System Libraries, or general-purpose tools or generally available free
+programs which are used unmodified in performing those activities but
+which are not part of the work. For example, Corresponding Source
+includes interface definition files associated with source files for
+the work, and the source code for shared libraries and dynamically
+linked subprograms that the work is specifically designed to require,
+such as by intimate data communication or control flow between those
+subprograms and other parts of the work.
+
+ The Corresponding Source need not include anything that users
+can regenerate automatically from other parts of the Corresponding
+Source.
+
+ The Corresponding Source for a work in source code form is that
+same work.
+
+ 2. Basic Permissions.
+
+ All rights granted under this License are granted for the term of
+copyright on the Program, and are irrevocable provided the stated
+conditions are met. This License explicitly affirms your unlimited
+permission to run the unmodified Program. The output from running a
+covered work is covered by this License only if the output, given its
+content, constitutes a covered work. This License acknowledges your
+rights of fair use or other equivalent, as provided by copyright law.
+
+ You may make, run and propagate covered works that you do not
+convey, without conditions so long as your license otherwise remains
+in force. You may convey covered works to others for the sole purpose
+of having them make modifications exclusively for you, or provide you
+with facilities for running those works, provided that you comply with
+the terms of this License in conveying all material for which you do
+not control copyright. Those thus making or running the covered works
+for you must do so exclusively on your behalf, under your direction
+and control, on terms that prohibit them from making any copies of
+your copyrighted material outside their relationship with you.
+
+ Conveying under any other circumstances is permitted solely under
+the conditions stated below. Sublicensing is not allowed; section 10
+makes it unnecessary.
+
+ 3. Protecting Users' Legal Rights From Anti-Circumvention Law.
+
+ No covered work shall be deemed part of an effective technological
+measure under any applicable law fulfilling obligations under article
+11 of the WIPO copyright treaty adopted on 20 December 1996, or
+similar laws prohibiting or restricting circumvention of such
+measures.
+
+ When you convey a covered work, you waive any legal power to forbid
+circumvention of technological measures to the extent such circumvention
+is effected by exercising rights under this License with respect to
+the covered work, and you disclaim any intention to limit operation or
+modification of the work as a means of enforcing, against the work's
+users, your or third parties' legal rights to forbid circumvention of
+technological measures.
+
+ 4. Conveying Verbatim Copies.
+
+ You may convey verbatim copies of the Program's source code as you
+receive it, in any medium, provided that you conspicuously and
+appropriately publish on each copy an appropriate copyright notice;
+keep intact all notices stating that this License and any
+non-permissive terms added in accord with section 7 apply to the code;
+keep intact all notices of the absence of any warranty; and give all
+recipients a copy of this License along with the Program.
+
+ You may charge any price or no price for each copy that you convey,
+and you may offer support or warranty protection for a fee.
+
+ 5. Conveying Modified Source Versions.
+
+ You may convey a work based on the Program, or the modifications to
+produce it from the Program, in the form of source code under the
+terms of section 4, provided that you also meet all of these conditions:
+
+ a) The work must carry prominent notices stating that you modified
+ it, and giving a relevant date.
+
+ b) The work must carry prominent notices stating that it is
+ released under this License and any conditions added under section
+ 7. This requirement modifies the requirement in section 4 to
+ "keep intact all notices".
+
+ c) You must license the entire work, as a whole, under this
+ License to anyone who comes into possession of a copy. This
+ License will therefore apply, along with any applicable section 7
+ additional terms, to the whole of the work, and all its parts,
+ regardless of how they are packaged. This License gives no
+ permission to license the work in any other way, but it does not
+ invalidate such permission if you have separately received it.
+
+ d) If the work has interactive user interfaces, each must display
+ Appropriate Legal Notices; however, if the Program has interactive
+ interfaces that do not display Appropriate Legal Notices, your
+ work need not make them do so.
+
+ A compilation of a covered work with other separate and independent
+works, which are not by their nature extensions of the covered work,
+and which are not combined with it such as to form a larger program,
+in or on a volume of a storage or distribution medium, is called an
+"aggregate" if the compilation and its resulting copyright are not
+used to limit the access or legal rights of the compilation's users
+beyond what the individual works permit. Inclusion of a covered work
+in an aggregate does not cause this License to apply to the other
+parts of the aggregate.
+
+ 6. Conveying Non-Source Forms.
+
+ You may convey a covered work in object code form under the terms
+of sections 4 and 5, provided that you also convey the
+machine-readable Corresponding Source under the terms of this License,
+in one of these ways:
+
+ a) Convey the object code in, or embodied in, a physical product
+ (including a physical distribution medium), accompanied by the
+ Corresponding Source fixed on a durable physical medium
+ customarily used for software interchange.
+
+ b) Convey the object code in, or embodied in, a physical product
+ (including a physical distribution medium), accompanied by a
+ written offer, valid for at least three years and valid for as
+ long as you offer spare parts or customer support for that product
+ model, to give anyone who possesses the object code either (1) a
+ copy of the Corresponding Source for all the software in the
+ product that is covered by this License, on a durable physical
+ medium customarily used for software interchange, for a price no
+ more than your reasonable cost of physically performing this
+ conveying of source, or (2) access to copy the
+ Corresponding Source from a network server at no charge.
+
+ c) Convey individual copies of the object code with a copy of the
+ written offer to provide the Corresponding Source. This
+ alternative is allowed only occasionally and noncommercially, and
+ only if you received the object code with such an offer, in accord
+ with subsection 6b.
+
+ d) Convey the object code by offering access from a designated
+ place (gratis or for a charge), and offer equivalent access to the
+ Corresponding Source in the same way through the same place at no
+ further charge. You need not require recipients to copy the
+ Corresponding Source along with the object code. If the place to
+ copy the object code is a network server, the Corresponding Source
+ may be on a different server (operated by you or a third party)
+ that supports equivalent copying facilities, provided you maintain
+ clear directions next to the object code saying where to find the
+ Corresponding Source. Regardless of what server hosts the
+ Corresponding Source, you remain obligated to ensure that it is
+ available for as long as needed to satisfy these requirements.
+
+ e) Convey the object code using peer-to-peer transmission, provided
+ you inform other peers where the object code and Corresponding
+ Source of the work are being offered to the general public at no
+ charge under subsection 6d.
+
+ A separable portion of the object code, whose source code is excluded
+from the Corresponding Source as a System Library, need not be
+included in conveying the object code work.
+
+ A "User Product" is either (1) a "consumer product", which means any
+tangible personal property which is normally used for personal, family,
+or household purposes, or (2) anything designed or sold for incorporation
+into a dwelling. In determining whether a product is a consumer product,
+doubtful cases shall be resolved in favor of coverage. For a particular
+product received by a particular user, "normally used" refers to a
+typical or common use of that class of product, regardless of the status
+of the particular user or of the way in which the particular user
+actually uses, or expects or is expected to use, the product. A product
+is a consumer product regardless of whether the product has substantial
+commercial, industrial or non-consumer uses, unless such uses represent
+the only significant mode of use of the product.
+
+ "Installation Information" for a User Product means any methods,
+procedures, authorization keys, or other information required to install
+and execute modified versions of a covered work in that User Product from
+a modified version of its Corresponding Source. The information must
+suffice to ensure that the continued functioning of the modified object
+code is in no case prevented or interfered with solely because
+modification has been made.
+
+ If you convey an object code work under this section in, or with, or
+specifically for use in, a User Product, and the conveying occurs as
+part of a transaction in which the right of possession and use of the
+User Product is transferred to the recipient in perpetuity or for a
+fixed term (regardless of how the transaction is characterized), the
+Corresponding Source conveyed under this section must be accompanied
+by the Installation Information. But this requirement does not apply
+if neither you nor any third party retains the ability to install
+modified object code on the User Product (for example, the work has
+been installed in ROM).
+
+ The requirement to provide Installation Information does not include a
+requirement to continue to provide support service, warranty, or updates
+for a work that has been modified or installed by the recipient, or for
+the User Product in which it has been modified or installed. Access to a
+network may be denied when the modification itself materially and
+adversely affects the operation of the network or violates the rules and
+protocols for communication across the network.
+
+ Corresponding Source conveyed, and Installation Information provided,
+in accord with this section must be in a format that is publicly
+documented (and with an implementation available to the public in
+source code form), and must require no special password or key for
+unpacking, reading or copying.
+
+ 7. Additional Terms.
+
+ "Additional permissions" are terms that supplement the terms of this
+License by making exceptions from one or more of its conditions.
+Additional permissions that are applicable to the entire Program shall
+be treated as though they were included in this License, to the extent
+that they are valid under applicable law. If additional permissions
+apply only to part of the Program, that part may be used separately
+under those permissions, but the entire Program remains governed by
+this License without regard to the additional permissions.
+
+ When you convey a copy of a covered work, you may at your option
+remove any additional permissions from that copy, or from any part of
+it. (Additional permissions may be written to require their own
+removal in certain cases when you modify the work.) You may place
+additional permissions on material, added by you to a covered work,
+for which you have or can give appropriate copyright permission.
+
+ Notwithstanding any other provision of this License, for material you
+add to a covered work, you may (if authorized by the copyright holders of
+that material) supplement the terms of this License with terms:
+
+ a) Disclaiming warranty or limiting liability differently from the
+ terms of sections 15 and 16 of this License; or
+
+ b) Requiring preservation of specified reasonable legal notices or
+ author attributions in that material or in the Appropriate Legal
+ Notices displayed by works containing it; or
+
+ c) Prohibiting misrepresentation of the origin of that material, or
+ requiring that modified versions of such material be marked in
+ reasonable ways as different from the original version; or
+
+ d) Limiting the use for publicity purposes of names of licensors or
+ authors of the material; or
+
+ e) Declining to grant rights under trademark law for use of some
+ trade names, trademarks, or service marks; or
+
+ f) Requiring indemnification of licensors and authors of that
+ material by anyone who conveys the material (or modified versions of
+ it) with contractual assumptions of liability to the recipient, for
+ any liability that these contractual assumptions directly impose on
+ those licensors and authors.
+
+ All other non-permissive additional terms are considered "further
+restrictions" within the meaning of section 10. If the Program as you
+received it, or any part of it, contains a notice stating that it is
+governed by this License along with a term that is a further
+restriction, you may remove that term. If a license document contains
+a further restriction but permits relicensing or conveying under this
+License, you may add to a covered work material governed by the terms
+of that license document, provided that the further restriction does
+not survive such relicensing or conveying.
+
+ If you add terms to a covered work in accord with this section, you
+must place, in the relevant source files, a statement of the
+additional terms that apply to those files, or a notice indicating
+where to find the applicable terms.
+
+ Additional terms, permissive or non-permissive, may be stated in the
+form of a separately written license, or stated as exceptions;
+the above requirements apply either way.
+
+ 8. Termination.
+
+ You may not propagate or modify a covered work except as expressly
+provided under this License. Any attempt otherwise to propagate or
+modify it is void, and will automatically terminate your rights under
+this License (including any patent licenses granted under the third
+paragraph of section 11).
+
+ However, if you cease all violation of this License, then your
+license from a particular copyright holder is reinstated (a)
+provisionally, unless and until the copyright holder explicitly and
+finally terminates your license, and (b) permanently, if the copyright
+holder fails to notify you of the violation by some reasonable means
+prior to 60 days after the cessation.
+
+ Moreover, your license from a particular copyright holder is
+reinstated permanently if the copyright holder notifies you of the
+violation by some reasonable means, this is the first time you have
+received notice of violation of this License (for any work) from that
+copyright holder, and you cure the violation prior to 30 days after
+your receipt of the notice.
+
+ Termination of your rights under this section does not terminate the
+licenses of parties who have received copies or rights from you under
+this License. If your rights have been terminated and not permanently
+reinstated, you do not qualify to receive new licenses for the same
+material under section 10.
+
+ 9. Acceptance Not Required for Having Copies.
+
+ You are not required to accept this License in order to receive or
+run a copy of the Program. Ancillary propagation of a covered work
+occurring solely as a consequence of using peer-to-peer transmission
+to receive a copy likewise does not require acceptance. However,
+nothing other than this License grants you permission to propagate or
+modify any covered work. These actions infringe copyright if you do
+not accept this License. Therefore, by modifying or propagating a
+covered work, you indicate your acceptance of this License to do so.
+
+ 10. Automatic Licensing of Downstream Recipients.
+
+ Each time you convey a covered work, the recipient automatically
+receives a license from the original licensors, to run, modify and
+propagate that work, subject to this License. You are not responsible
+for enforcing compliance by third parties with this License.
+
+ An "entity transaction" is a transaction transferring control of an
+organization, or substantially all assets of one, or subdividing an
+organization, or merging organizations. If propagation of a covered
+work results from an entity transaction, each party to that
+transaction who receives a copy of the work also receives whatever
+licenses to the work the party's predecessor in interest had or could
+give under the previous paragraph, plus a right to possession of the
+Corresponding Source of the work from the predecessor in interest, if
+the predecessor has it or can get it with reasonable efforts.
+
+ You may not impose any further restrictions on the exercise of the
+rights granted or affirmed under this License. For example, you may
+not impose a license fee, royalty, or other charge for exercise of
+rights granted under this License, and you may not initiate litigation
+(including a cross-claim or counterclaim in a lawsuit) alleging that
+any patent claim is infringed by making, using, selling, offering for
+sale, or importing the Program or any portion of it.
+
+ 11. Patents.
+
+ A "contributor" is a copyright holder who authorizes use under this
+License of the Program or a work on which the Program is based. The
+work thus licensed is called the contributor's "contributor version".
+
+ A contributor's "essential patent claims" are all patent claims
+owned or controlled by the contributor, whether already acquired or
+hereafter acquired, that would be infringed by some manner, permitted
+by this License, of making, using, or selling its contributor version,
+but do not include claims that would be infringed only as a
+consequence of further modification of the contributor version. For
+purposes of this definition, "control" includes the right to grant
+patent sublicenses in a manner consistent with the requirements of
+this License.
+
+ Each contributor grants you a non-exclusive, worldwide, royalty-free
+patent license under the contributor's essential patent claims, to
+make, use, sell, offer for sale, import and otherwise run, modify and
+propagate the contents of its contributor version.
+
+ In the following three paragraphs, a "patent license" is any express
+agreement or commitment, however denominated, not to enforce a patent
+(such as an express permission to practice a patent or covenant not to
+sue for patent infringement). To "grant" such a patent license to a
+party means to make such an agreement or commitment not to enforce a
+patent against the party.
+
+ If you convey a covered work, knowingly relying on a patent license,
+and the Corresponding Source of the work is not available for anyone
+to copy, free of charge and under the terms of this License, through a
+publicly available network server or other readily accessible means,
+then you must either (1) cause the Corresponding Source to be so
+available, or (2) arrange to deprive yourself of the benefit of the
+patent license for this particular work, or (3) arrange, in a manner
+consistent with the requirements of this License, to extend the patent
+license to downstream recipients. "Knowingly relying" means you have
+actual knowledge that, but for the patent license, your conveying the
+covered work in a country, or your recipient's use of the covered work
+in a country, would infringe one or more identifiable patents in that
+country that you have reason to believe are valid.
+
+ If, pursuant to or in connection with a single transaction or
+arrangement, you convey, or propagate by procuring conveyance of, a
+covered work, and grant a patent license to some of the parties
+receiving the covered work authorizing them to use, propagate, modify
+or convey a specific copy of the covered work, then the patent license
+you grant is automatically extended to all recipients of the covered
+work and works based on it.
+
+ A patent license is "discriminatory" if it does not include within
+the scope of its coverage, prohibits the exercise of, or is
+conditioned on the non-exercise of one or more of the rights that are
+specifically granted under this License. You may not convey a covered
+work if you are a party to an arrangement with a third party that is
+in the business of distributing software, under which you make payment
+to the third party based on the extent of your activity of conveying
+the work, and under which the third party grants, to any of the
+parties who would receive the covered work from you, a discriminatory
+patent license (a) in connection with copies of the covered work
+conveyed by you (or copies made from those copies), or (b) primarily
+for and in connection with specific products or compilations that
+contain the covered work, unless you entered into that arrangement,
+or that patent license was granted, prior to 28 March 2007.
+
+ Nothing in this License shall be construed as excluding or limiting
+any implied license or other defenses to infringement that may
+otherwise be available to you under applicable patent law.
+
+ 12. No Surrender of Others' Freedom.
+
+ If conditions are imposed on you (whether by court order, agreement or
+otherwise) that contradict the conditions of this License, they do not
+excuse you from the conditions of this License. If you cannot convey a
+covered work so as to satisfy simultaneously your obligations under this
+License and any other pertinent obligations, then as a consequence you may
+not convey it at all. For example, if you agree to terms that obligate you
+to collect a royalty for further conveying from those to whom you convey
+the Program, the only way you could satisfy both those terms and this
+License would be to refrain entirely from conveying the Program.
+
+ 13. Remote Network Interaction; Use with the GNU General Public License.
+
+ Notwithstanding any other provision of this License, if you modify the
+Program, your modified version must prominently offer all users
+interacting with it remotely through a computer network (if your version
+supports such interaction) an opportunity to receive the Corresponding
+Source of your version by providing access to the Corresponding Source
+from a network server at no charge, through some standard or customary
+means of facilitating copying of software. This Corresponding Source
+shall include the Corresponding Source for any work covered by version 3
+of the GNU General Public License that is incorporated pursuant to the
+following paragraph.
+
+ Notwithstanding any other provision of this License, you have
+permission to link or combine any covered work with a work licensed
+under version 3 of the GNU General Public License into a single
+combined work, and to convey the resulting work. The terms of this
+License will continue to apply to the part which is the covered work,
+but the work with which it is combined will remain governed by version
+3 of the GNU General Public License.
+
+ 14. Revised Versions of this License.
+
+ The Free Software Foundation may publish revised and/or new versions of
+the GNU Affero General Public License from time to time. Such new versions
+will be similar in spirit to the present version, but may differ in detail to
+address new problems or concerns.
+
+ Each version is given a distinguishing version number. If the
+Program specifies that a certain numbered version of the GNU Affero General
+Public License "or any later version" applies to it, you have the
+option of following the terms and conditions either of that numbered
+version or of any later version published by the Free Software
+Foundation. If the Program does not specify a version number of the
+GNU Affero General Public License, you may choose any version ever published
+by the Free Software Foundation.
+
+ If the Program specifies that a proxy can decide which future
+versions of the GNU Affero General Public License can be used, that proxy's
+public statement of acceptance of a version permanently authorizes you
+to choose that version for the Program.
+
+ Later license versions may give you additional or different
+permissions. However, no additional obligations are imposed on any
+author or copyright holder as a result of your choosing to follow a
+later version.
+
+ 15. Disclaimer of Warranty.
+
+ THERE IS NO WARRANTY FOR THE PROGRAM, TO THE EXTENT PERMITTED BY
+APPLICABLE LAW. EXCEPT WHEN OTHERWISE STATED IN WRITING THE COPYRIGHT
+HOLDERS AND/OR OTHER PARTIES PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY
+OF ANY KIND, EITHER EXPRESSED OR IMPLIED, INCLUDING, BUT NOT LIMITED TO,
+THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+PURPOSE. THE ENTIRE RISK AS TO THE QUALITY AND PERFORMANCE OF THE PROGRAM
+IS WITH YOU. SHOULD THE PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF
+ALL NECESSARY SERVICING, REPAIR OR CORRECTION.
+
+ 16. Limitation of Liability.
+
+ IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING
+WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MODIFIES AND/OR CONVEYS
+THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES, INCLUDING ANY
+GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING OUT OF THE
+USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED TO LOSS OF
+DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY YOU OR THIRD
+PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER PROGRAMS),
+EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE POSSIBILITY OF
+SUCH DAMAGES.
+
+ 17. Interpretation of Sections 15 and 16.
+
+ If the disclaimer of warranty and limitation of liability provided
+above cannot be given local legal effect according to their terms,
+reviewing courts shall apply local law that most closely approximates
+an absolute waiver of all civil liability in connection with the
+Program, unless a warranty or assumption of liability accompanies a
+copy of the Program in return for a fee.
+
+ END OF TERMS AND CONDITIONS
+
+ How to Apply These Terms to Your New Programs
+
+ If you develop a new program, and you want it to be of the greatest
+possible use to the public, the best way to achieve this is to make it
+free software which everyone can redistribute and change under these terms.
+
+ To do so, attach the following notices to the program. It is safest
+to attach them to the start of each source file to most effectively
+state the exclusion of warranty; and each file should have at least
+the "copyright" line and a pointer to where the full notice is found.
+
+ <one line to give the program's name and a brief idea of what it does.>
+ Copyright (C) <year> <name of author>
+
+ This program is free software: you can redistribute it and/or modify
+ it under the terms of the GNU Affero General Public License as published by
+ the Free Software Foundation, either version 3 of the License, or
+ (at your option) any later version.
+
+ This program is distributed in the hope that it will be useful,
+ but WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ GNU Affero General Public License for more details.
+
+ You should have received a copy of the GNU Affero General Public License
+ along with this program. If not, see <http://www.gnu.org/licenses/>.
+
+Also add information on how to contact you by electronic and paper mail.
+
+ If your software can interact with users remotely through a computer
+network, you should also make sure that it provides a way for users to
+get its source. For example, if your program is a web application, its
+interface could display a "Source" link that leads users to an archive
+of the code. There are many ways you could offer source, and different
+solutions will be better for different programs; see section 13 for the
+specific requirements.
+
+ You should also get your employer (if you work as a programmer) or school,
+if any, to sign a "copyright disclaimer" for the program, if necessary.
+For more information on this, and how to apply and follow the GNU AGPL, see
+<http://www.gnu.org/licenses/>.
/config/environments/development.rb
/config/environments/test.rb
/config/environments/production.rb
+/config/application.yml
/config/piwik.yml
gem 'uglifier', '>= 1.0.3'
end
+group :test do
+ gem 'rvm-capistrano'
+ gem 'selenium-webdriver'
+ gem 'capybara'
+ gem 'poltergeist'
+ gem 'headless'
+end
+
gem 'jquery-rails'
gem 'bootstrap-sass', '~> 3.1.0'
gem 'bootstrap-x-editable-rails'
# To use debugger
#gem 'byebug'
-gem 'rvm-capistrano', :group => :test
-
gem 'passenger', :group => :production
gem 'andand'
gem 'RedCloth'
gem 'piwik_analytics'
gem 'httpclient'
gem 'themes_for_rails'
-gem "deep_merge", :require => 'deep_merge/rails_compat'
\ No newline at end of file
+gem "deep_merge", :require => 'deep_merge/rails_compat'
net-sftp (>= 2.0.0)
net-ssh (>= 2.0.14)
net-ssh-gateway (>= 1.1.0)
+ capybara (2.2.1)
+ mime-types (>= 1.16)
+ nokogiri (>= 1.3.3)
+ rack (>= 1.0.0)
+ rack-test (>= 0.5.4)
+ xpath (~> 2.0)
+ childprocess (0.5.1)
+ ffi (~> 1.0, >= 1.0.11)
+ cliver (0.3.2)
coffee-rails (3.2.2)
coffee-script (>= 2.2.0)
railties (~> 3.2.0)
deep_merge (1.0.1)
erubis (2.7.0)
execjs (2.0.2)
+ ffi (1.9.3)
+ headless (1.0.1)
highline (1.6.20)
hike (1.2.3)
httpclient (2.3.4.1)
mime-types (~> 1.16)
treetop (~> 1.4.8)
mime-types (1.25)
+ mini_portile (0.5.2)
multi_json (1.8.2)
net-scp (1.1.2)
net-ssh (>= 2.6.5)
net-ssh (2.7.0)
net-ssh-gateway (1.2.0)
net-ssh (>= 2.6.5)
+ nokogiri (1.6.1)
+ mini_portile (~> 0.5.0)
oj (2.1.7)
passenger (4.0.23)
daemon_controller (>= 1.1.0)
actionpack
activesupport
rails (>= 3.0.0)
+ poltergeist (1.5.0)
+ capybara (~> 2.1)
+ cliver (~> 0.3.1)
+ multi_json (~> 1.0)
+ websocket-driver (>= 0.2.0)
polyglot (0.3.3)
rack (1.4.5)
rack-cache (1.2)
rdoc (3.12.2)
json (~> 1.4)
ref (1.0.5)
+ rubyzip (1.1.0)
rvm-capistrano (1.5.1)
capistrano (~> 2.15.4)
sass (3.2.12)
railties (~> 3.2.0)
sass (>= 3.1.10)
tilt (~> 1.3)
+ selenium-webdriver (2.40.0)
+ childprocess (>= 0.5.0)
+ multi_json (~> 1.0)
+ rubyzip (~> 1.0)
+ websocket (~> 1.0.4)
sprockets (2.2.2)
hike (~> 1.2)
multi_json (~> 1.0)
uglifier (2.3.1)
execjs (>= 0.3.0)
json (>= 1.8.0)
+ websocket (1.0.7)
+ websocket-driver (0.3.2)
+ xpath (2.0.0)
+ nokogiri (~> 1.3)
PLATFORMS
ruby
andand
bootstrap-sass (~> 3.1.0)
bootstrap-x-editable-rails
+ capybara
coffee-rails (~> 3.2.0)
deep_merge
+ headless
httpclient
jquery-rails
less
oj
passenger
piwik_analytics
+ poltergeist
rails (~> 3.2.0)
rvm-capistrano
sass
sass-rails (~> 3.2.0)
+ selenium-webdriver
sqlite3
themes_for_rails
therubyracer
+++ /dev/null
-== Welcome to Rails
-
-Rails is a web-application framework that includes everything needed to create
-database-backed web applications according to the Model-View-Control pattern.
-
-This pattern splits the view (also called the presentation) into "dumb"
-templates that are primarily responsible for inserting pre-built data in between
-HTML tags. The model contains the "smart" domain objects (such as Account,
-Product, Person, Post) that holds all the business logic and knows how to
-persist themselves to a database. The controller handles the incoming requests
-(such as Save New Account, Update Product, Show Post) by manipulating the model
-and directing data to the view.
-
-In Rails, the model is handled by what's called an object-relational mapping
-layer entitled Active Record. This layer allows you to present the data from
-database rows as objects and embellish these data objects with business logic
-methods. You can read more about Active Record in
-link:files/vendor/rails/activerecord/README.html.
-
-The controller and view are handled by the Action Pack, which handles both
-layers by its two parts: Action View and Action Controller. These two layers
-are bundled in a single package due to their heavy interdependence. This is
-unlike the relationship between the Active Record and Action Pack that is much
-more separate. Each of these packages can be used independently outside of
-Rails. You can read more about Action Pack in
-link:files/vendor/rails/actionpack/README.html.
-
-
-== Getting Started
-
-1. At the command prompt, create a new Rails application:
- <tt>rails new myapp</tt> (where <tt>myapp</tt> is the application name)
-
-2. Change directory to <tt>myapp</tt> and start the web server:
- <tt>cd myapp; rails server</tt> (run with --help for options)
-
-3. Go to http://localhost:3000/ and you'll see:
- "Welcome aboard: You're riding Ruby on Rails!"
-
-4. Follow the guidelines to start developing your application. You can find
-the following resources handy:
-
-* The Getting Started Guide: http://guides.rubyonrails.org/getting_started.html
-* Ruby on Rails Tutorial Book: http://www.railstutorial.org/
-
-
-== Debugging Rails
-
-Sometimes your application goes wrong. Fortunately there are a lot of tools that
-will help you debug it and get it back on the rails.
-
-First area to check is the application log files. Have "tail -f" commands
-running on the server.log and development.log. Rails will automatically display
-debugging and runtime information to these files. Debugging info will also be
-shown in the browser on requests from 127.0.0.1.
-
-You can also log your own messages directly into the log file from your code
-using the Ruby logger class from inside your controllers. Example:
-
- class WeblogController < ActionController::Base
- def destroy
- @weblog = Weblog.find(params[:id])
- @weblog.destroy
- logger.info("#{Time.now} Destroyed Weblog ID ##{@weblog.id}!")
- end
- end
-
-The result will be a message in your log file along the lines of:
-
- Mon Oct 08 14:22:29 +1000 2007 Destroyed Weblog ID #1!
-
-More information on how to use the logger is at http://www.ruby-doc.org/core/
-
-Also, Ruby documentation can be found at http://www.ruby-lang.org/. There are
-several books available online as well:
-
-* Programming Ruby: http://www.ruby-doc.org/docs/ProgrammingRuby/ (Pickaxe)
-* Learn to Program: http://pine.fm/LearnToProgram/ (a beginners guide)
-
-These two books will bring you up to speed on the Ruby language and also on
-programming in general.
-
-
-== Debugger
-
-Debugger support is available through the debugger command when you start your
-Mongrel or WEBrick server with --debugger. This means that you can break out of
-execution at any point in the code, investigate and change the model, and then,
-resume execution! You need to install ruby-debug to run the server in debugging
-mode. With gems, use <tt>sudo gem install ruby-debug</tt>. Example:
-
- class WeblogController < ActionController::Base
- def index
- @posts = Post.all
- debugger
- end
- end
-
-So the controller will accept the action, run the first line, then present you
-with a IRB prompt in the server window. Here you can do things like:
-
- >> @posts.inspect
- => "[#<Post:0x14a6be8
- @attributes={"title"=>nil, "body"=>nil, "id"=>"1"}>,
- #<Post:0x14a6620
- @attributes={"title"=>"Rails", "body"=>"Only ten..", "id"=>"2"}>]"
- >> @posts.first.title = "hello from a debugger"
- => "hello from a debugger"
-
-...and even better, you can examine how your runtime objects actually work:
-
- >> f = @posts.first
- => #<Post:0x13630c4 @attributes={"title"=>nil, "body"=>nil, "id"=>"1"}>
- >> f.
- Display all 152 possibilities? (y or n)
-
-Finally, when you're ready to resume execution, you can enter "cont".
-
-
-== Console
-
-The console is a Ruby shell, which allows you to interact with your
-application's domain model. Here you'll have all parts of the application
-configured, just like it is when the application is running. You can inspect
-domain models, change values, and save to the database. Starting the script
-without arguments will launch it in the development environment.
-
-To start the console, run <tt>rails console</tt> from the application
-directory.
-
-Options:
-
-* Passing the <tt>-s, --sandbox</tt> argument will rollback any modifications
- made to the database.
-* Passing an environment name as an argument will load the corresponding
- environment. Example: <tt>rails console production</tt>.
-
-To reload your controllers and models after launching the console run
-<tt>reload!</tt>
-
-More information about irb can be found at:
-link:http://www.rubycentral.org/pickaxe/irb.html
-
-
-== dbconsole
-
-You can go to the command line of your database directly through <tt>rails
-dbconsole</tt>. You would be connected to the database with the credentials
-defined in database.yml. Starting the script without arguments will connect you
-to the development database. Passing an argument will connect you to a different
-database, like <tt>rails dbconsole production</tt>. Currently works for MySQL,
-PostgreSQL and SQLite 3.
-
-== Description of Contents
-
-The default directory structure of a generated Ruby on Rails application:
-
- |-- app
- | |-- assets
- | |-- images
- | |-- javascripts
- | `-- stylesheets
- | |-- controllers
- | |-- helpers
- | |-- mailers
- | |-- models
- | `-- views
- | `-- layouts
- |-- config
- | |-- environments
- | |-- initializers
- | `-- locales
- |-- db
- |-- doc
- |-- lib
- | `-- tasks
- |-- log
- |-- public
- |-- script
- |-- test
- | |-- fixtures
- | |-- functional
- | |-- integration
- | |-- performance
- | `-- unit
- |-- tmp
- | |-- cache
- | |-- pids
- | |-- sessions
- | `-- sockets
- `-- vendor
- |-- assets
- `-- stylesheets
- `-- plugins
-
-app
- Holds all the code that's specific to this particular application.
-
-app/assets
- Contains subdirectories for images, stylesheets, and JavaScript files.
-
-app/controllers
- Holds controllers that should be named like weblogs_controller.rb for
- automated URL mapping. All controllers should descend from
- ApplicationController which itself descends from ActionController::Base.
-
-app/models
- Holds models that should be named like post.rb. Models descend from
- ActiveRecord::Base by default.
-
-app/views
- Holds the template files for the view that should be named like
- weblogs/index.html.erb for the WeblogsController#index action. All views use
- eRuby syntax by default.
-
-app/views/layouts
- Holds the template files for layouts to be used with views. This models the
- common header/footer method of wrapping views. In your views, define a layout
- using the <tt>layout :default</tt> and create a file named default.html.erb.
- Inside default.html.erb, call <% yield %> to render the view using this
- layout.
-
-app/helpers
- Holds view helpers that should be named like weblogs_helper.rb. These are
- generated for you automatically when using generators for controllers.
- Helpers can be used to wrap functionality for your views into methods.
-
-config
- Configuration files for the Rails environment, the routing map, the database,
- and other dependencies.
-
-db
- Contains the database schema in schema.rb. db/migrate contains all the
- sequence of Migrations for your schema.
-
-doc
- This directory is where your application documentation will be stored when
- generated using <tt>rake doc:app</tt>
-
-lib
- Application specific libraries. Basically, any kind of custom code that
- doesn't belong under controllers, models, or helpers. This directory is in
- the load path.
-
-public
- The directory available for the web server. Also contains the dispatchers and the
- default HTML files. This should be set as the DOCUMENT_ROOT of your web
- server.
-
-script
- Helper scripts for automation and generation.
-
-test
- Unit and functional tests along with fixtures. When using the rails generate
- command, template test files will be generated for you and placed in this
- directory.
-
-vendor
- External libraries that the application depends on. Also includes the plugins
- subdirectory. If the app has frozen rails, those gems also go here, under
- vendor/rails/. This directory is in the load path.
--- /dev/null
+h1. Developing Workbench
+
+This document includes information to help developers who would like to contribute to Workbench. If you just want to install it, please refer to our "Workbench installation guide":http://doc.arvados.org/install/install-workbench-app.html.
+
+h2. Running tests
+
+The Workbench application includes a series of integration tests. When you run these, it starts the API server in a test environment, with all of its fixtures loaded, then tests Workbench by starting that server and making requests against it.
+
+In order for this to work, you must have Firefox installed (or Iceweasel, if you're running Debian), as well as the X Virtual Frame Buffer driver.
+
+<pre>
+$ sudo apt-get install iceweasel xvfb
+</pre>
+
+If you install the Workbench Bundle in deployment mode, you must also install the API server Bundle in deployment mode, and vice versa. If your Bundle installs have mismatched modes, the integration tests will fail with "Gem not found" errors.
+
+h2. Writing tests
+
+Integration tests are written with Capybara, which drives a fully-featured Web browser to interact with Workbench exactly as a user would.
+
+If your test requires JavaScript support, your test method should start with the line @Capybara.current_driver = Capybara.javascript_driver@. Otherwise, Capybara defaults to a simpler browser for speed.
+
+In most tests, you can directly call "Capybara's Session methods":http://rubydoc.info/github/jnicklas/capybara/Capybara/Session to drive the browser and check its state. If you need finer-grained control, refer to the "full Capybara documentation":http://rubydoc.info/github/jnicklas/capybara/Capybara.
//= require bootstrap/tooltip
//= require bootstrap/popover
//= require bootstrap/collapse
+//= require bootstrap/modal
//= require bootstrap3-editable/bootstrap-editable
//= require_tree .
{dataType: 'json',
type: $(this).attr('data-remote-method'),
data: {
- 'link[head_kind]': 'arvados#collection',
'link[head_uuid]': tag_head_uuid,
'link[link_class]': 'tag',
'link[name]': new_tag
});
}
}
-
+
var fixer = new HeaderRowFixer('.table-fixed-header-row');
fixer.duplicateTheadTr();
fixer.fixThead();
$.fn.editable.defaults.ajaxOptions = {type: 'put', dataType: 'json'};
$.fn.editable.defaults.send = 'always';
+
+// Default for editing is popup. I experimented with inline which is a little
+// nicer in that it shows up right under the mouse instead of nearby. However,
+// the inline box is taller than the regular content, which causes the page
+// layout to shift unless we make the table rows tall, which leaves a lot of
+// wasted space when not editing. Also inline can get cut off if the page is
+// too narrow, when the popup box will just move to do the right thing.
//$.fn.editable.defaults.mode = 'inline';
+
$.fn.editable.defaults.params = function (params) {
var a = {};
var key = params.pk.key;
a[key][params.name] = params.value;
return a;
};
+
+$.fn.editable.defaults.validate = function (value) {
+ if (value == "***invalid***") {
+ return "Invalid selection";
+ }
+}
+
+$.fn.editabletypes.text.defaults.tpl = '<input type="text" name="editable-text">'
+
+$.fn.editableform.buttons = '\
+<button type="submit" class="btn btn-primary btn-sm editable-submit" \
+ id="editable-submit"><i class="glyphicon glyphicon-ok"></i></button>\
+<button type="button" class="btn btn-default btn-sm editable-cancel" \
+ id="editable-cancel"><i class="glyphicon glyphicon-remove"></i></button>\
+'
};
var remove_selection_click = function(e) {
- //remove_selection($(this).attr('name'));
remove_selection($(this).val());
};
var update_count = function(e) {
var lst = get_selection_list();
$("#persistent-selection-count").text(lst.length);
-
if (lst.length > 0) {
- $('#persistent-selection-list').html('<li><a href="#" class="btn pull-right" id="clear_selections_button">Clear selections</a></li>'
- +'<li class="notification"><table style="width: 100%"></table></li>');
+ $('#selection-form-content').html(
+ '<li><a href="#" id="clear_selections_button">Clear selections</a></li>'
+ + '<li><input type="submit" name="combine_selected_files_into_collection" '
+ + ' id="combine_selected_files_into_collection" '
+ + ' value="Combine selected collections and files into a new collection" /></li>'
+ + '<li class="notification"><table style="width: 100%"></table></li>');
+
for (var i = 0; i < lst.length; i++) {
- $('#persistent-selection-list > li > table').append("<tr>"
+ $('#selection-form-content > li > table').append("<tr>"
+ "<td>"
- + "<form>"
- + "<input class='remove-selection' type='checkbox' value='" + lst[i].uuid + "' checked='true'></input>"
- + "</form>"
+ + "<input class='remove-selection' name='selection[]' type='checkbox' value='" + lst[i].uuid + "' checked='true' data-stoppropagation='true' />"
+ "</td>"
+ "<td>"
- + "<span style='padding-left: 1em'><a href=\"" + lst[i].href + "\">" + lst[i].name + "</a></span>"
+ + "<div style='padding-left: 1em'><a href=\"" + lst[i].href + "\">" + lst[i].name + "</a></div>"
+ "</td>"
+ "<td style=\"vertical-align: top\">"
+ "</tr>");
}
} else {
- $('#persistent-selection-list').html("<li class='notification empty'>No selections.</li>");
+ $('#selection-form-content').html("<li class='notification empty'>No selections.</li>");
}
var checkboxes = $('.persistent-selection:checkbox');
$(window).on('load storage', update_count);
+
+ $('#selection-form-content').on("click", function(e) {
+ e.stopPropagation();
+ });
});
add_form_selection_sources = null;
if (get_selection_list) {
var lst = get_selection_list();
if (lst.length > 0) {
- ret.push({text: "--- Selections ---", value: ""});
+ var text = "― Selections ―";
+ var span = document.createElement('span');
+ span.innerHTML = text;
+ ret.push({text: span.innerHTML, value: "***invalid***"});
for (var i = 0; i < lst.length; i++) {
if (lst[i].type == type) {
}
}
}
- ret.push({text: "--- Recent ---", value: ""});
+
+ var text = "― Recent ―";
+ var span = document.createElement('span');
+ span.innerHTML = text;
+ ret.push({text: span.innerHTML, value: "***invalid***"});
var t = form_selection_sources[type];
for (var key in t) {
}
.inline-progress-container {
- width: 100px;
+ width: 100%;
display:inline-block;
}
overflow-y: auto;
}
-#persistent-selection-list {
- width: 500px;
-}
-
-#persistent-selection-list li table tr {
- border-top: 1px solid rgb(221, 221, 221);
-}
// Place all the styles related to the PipelineTemplates controller here.
// They will automatically be included in application.css.
// You can use Sass (SCSS) here: http://sass-lang.com/
+
+.pipeline_color_legend {
+ padding-left: 1em;
+ padding-right: 1em;
+}
+
+table.pipeline-components-table {
+ width: 100%;
+ table-layout: fixed;
+ overflow: hidden;
+}
+
+table.pipeline-components-table thead th {
+ text-align: bottom;
+}
+table.pipeline-components-table div.progress {
+ margin-bottom: 0;
+}
+
+table.pipeline-components-table td {
+ overflow: hidden;
+ text-overflow: ellipsis;
+}
+
+td.required {
+ background: #ffdddd;
+}
--- /dev/null
+#persistent-selection-list {
+ width: 500px;
+}
+
+#selection-form-content > li > a, #selection-form-content > li > input {
+ display: block;
+ padding: 3px 20px;
+ clear: both;
+ font-weight: normal;
+ line-height: 1.42857;
+ color: rgb(51, 51, 51);
+ white-space: nowrap;
+ border: none;
+ background: transparent;
+ width: 100%;
+ text-align: left;
+}
+
+#selection-form-content li table tr {
+ padding: 3px 20px;
+ line-height: 1.42857;
+ border-top: 1px solid rgb(221, 221, 221);
+}
+
+#selection-form-content a:hover, #selection-form-content a:focus, #selection-form-content input:hover, #selection-form-content input:focus, #selection-form-content tr:hover {
+ text-decoration: none;
+ color: rgb(38, 38, 38);
+ background-color: whitesmoke;
+}
\ No newline at end of file
--- /dev/null
+class ActionsController < ApplicationController
+
+ skip_before_filter :find_object_by_uuid, only: :post
+
+ def combine_selected_files_into_collection
+ lst = []
+ files = []
+ params["selection"].each do |s|
+ m = CollectionsHelper.match(s)
+ if m and m[1] and m[2]
+ lst.append(m[1] + m[2])
+ files.append(m)
+ end
+ end
+
+ collections = Collection.where(uuid: lst)
+
+ chash = {}
+ collections.each do |c|
+ c.reload()
+ chash[c.uuid] = c
+ end
+
+ combined = ""
+ files.each do |m|
+ mt = chash[m[1]+m[2]].manifest_text
+ if m[4]
+ IO.popen(['arv-normalize', '--extract', m[4][1..-1]], 'w+b') do |io|
+ io.write mt
+ io.close_write
+ while buf = io.read(2**20)
+ combined += buf
+ end
+ end
+ else
+ combined += chash[m[1]+m[2]].manifest_text
+ end
+ end
+
+ normalized = ''
+ IO.popen(['arv-normalize'], 'w+b') do |io|
+ io.write combined
+ io.close_write
+ while buf = io.read(2**20)
+ normalized += buf
+ end
+ end
+
+ require 'digest/md5'
+
+ d = Digest::MD5.new()
+ d << normalized
+ newuuid = "#{d.hexdigest}+#{normalized.length}"
+
+ env = Hash[ENV].
+ merge({
+ 'ARVADOS_API_HOST' =>
+ $arvados_api_client.arvados_v1_base.
+ sub(/\/arvados\/v1/, '').
+ sub(/^https?:\/\//, ''),
+ 'ARVADOS_API_TOKEN' => Thread.current[:arvados_api_token],
+ 'ARVADOS_API_HOST_INSECURE' =>
+ Rails.configuration.arvados_insecure_https ? 'true' : 'false'
+ })
+
+ IO.popen([env, 'arv-put', '--raw'], 'w+b') do |io|
+ io.write normalized
+ io.close_write
+ while buf = io.read(2**20)
+
+ end
+ end
+
+ newc = Collection.new({:uuid => newuuid, :manifest_text => normalized})
+ newc.save!
+
+ chash.each do |k,v|
+ l = Link.new({
+ tail_uuid: k,
+ head_uuid: newuuid,
+ link_class: "provenance",
+ name: "provided"
+ })
+ l.save!
+ end
+
+ redirect_to controller: 'collections', action: :show, id: newc.uuid
+ end
+
+ def post
+ if params["combine_selected_files_into_collection"]
+ combine_selected_files_into_collection
+ else
+ redirect_to :back
+ end
+ end
+end
class ApiClientAuthorizationsController < ApplicationController
def index
- @objects = model_class.all.to_ary.reject do |x|
+ m = model_class.all
+ items_available = m.items_available
+ offset = m.result_offset
+ limit = m.result_limit
+ filtered = m.to_ary.reject do |x|
x.api_client_id == 0 or (x.expires_at and x.expires_at < Time.now) rescue false
end
+ ArvadosApiClient::patch_paging_vars(filtered, items_available, offset, limit)
+ @objects = ArvadosResourceList.new(ApiClientAuthorization)
+ @objects.results= filtered
super
end
end
def render_error(opts)
+ opts = {status: 500}.merge opts
respond_to do |f|
# json must come before html here, so it gets used as the
# default format when js is requested by the client. This lets
end
def index
- @objects ||= model_class.limit(200).all
+ if params[:limit]
+ limit = params[:limit].to_i
+ else
+ limit = 200
+ end
+
+ if params[:offset]
+ offset = params[:offset].to_i
+ else
+ offset = 0
+ end
+
+ @objects ||= model_class.limit(limit).offset(offset).all
respond_to do |f|
f.json { render json: @objects }
f.html { render }
def breadcrumb_page_name
(@breadcrumb_page_name ||
- (@object.friendly_link_name if @object.respond_to? :friendly_link_name))
+ (@object.friendly_link_name if @object.respond_to? :friendly_link_name) ||
+ action_name)
end
def index_pane_list
}
}
- @@notification_tests.push lambda { |controller, current_user|
- Job.limit(1).where(created_by: current_user.uuid).each do
- return nil
- end
- return lambda { |view|
- view.render partial: 'notifications/jobs_notification'
- }
- }
+ #@@notification_tests.push lambda { |controller, current_user|
+ # Job.limit(1).where(created_by: current_user.uuid).each do
+ # return nil
+ # end
+ # return lambda { |view|
+ # view.render partial: 'notifications/jobs_notification'
+ # }
+ #}
@@notification_tests.push lambda { |controller, current_user|
Collection.limit(1).where(created_by: current_user.uuid).each do
Collection.where(any: ['contains', params[:search]])).
uniq { |c| c.uuid }
else
- @collections = Collection.limit(100)
+ if params[:limit]
+ limit = params[:limit].to_i
+ else
+ limit = 100
+ end
+
+ if params[:offset]
+ offset = params[:offset].to_i
+ else
+ offset = 0
+ end
+
+ @collections = Collection.limit(limit).offset(offset)
end
@links = Link.limit(1000).
where(head_uuid: @collections.collect(&:uuid))
Link.where(tail_uuid: @sourcedata.keys).each do |link|
if link.link_class == 'data_origin'
@sourcedata[link.tail_uuid][:data_origins] ||= []
- @sourcedata[link.tail_uuid][:data_origins] << [link.name, link.head_kind, link.head_uuid]
+ @sourcedata[link.tail_uuid][:data_origins] << [link.name, link.head_uuid]
end
end
Collection.where(uuid: @sourcedata.keys).each do |collection|
@sourcedata[collection.uuid][:collection] = collection
end
end
-
+
Collection.where(uuid: @object.uuid).each do |u|
- @prov_svg = ProvenanceHelper::create_provenance_graph u.provenance, "provenance_svg", {:direction => :top_down, :combine_jobs => :script_only} rescue nil
- @used_by_svg = ProvenanceHelper::create_provenance_graph u.used_by, "used_by_svg", {:direction => :top_down, :combine_jobs => :script_only, :pdata_only => true} rescue nil
+ puts request
+ @prov_svg = ProvenanceHelper::create_provenance_graph(u.provenance, "provenance_svg",
+ {:request => request,
+ :direction => :bottom_up,
+ :combine_jobs => :script_only}) rescue nil
+ @used_by_svg = ProvenanceHelper::create_provenance_graph(u.used_by, "used_by_svg",
+ {:request => request,
+ :direction => :top_down,
+ :combine_jobs => :script_only,
+ :pdata_only => true}) rescue nil
end
end
nodes << c
end
- @svg = ProvenanceHelper::create_provenance_graph nodes, "provenance_svg", {:all_script_parameters => true, :script_version_nodes => true}
+ @svg = ProvenanceHelper::create_provenance_graph nodes, "provenance_svg", {
+ :request => request,
+ :all_script_parameters => true,
+ :script_version_nodes => true}
end
def index
class KeepDisksController < ApplicationController
+ def create
+ defaults = { is_readable: true, is_writable: true }
+ @object = KeepDisk.new defaults.merge(params[:keep_disk] || {})
+ super
+ end
end
p.components.each do |k, v|
j = v[:job] || next
+ # The graph is interested in whether the component is
+ # indicated as persistent, more than whether the job
+ # satisfying it (which could have been reused, or someone
+ # else's) is.
+ j[:output_is_persistent] = v[:output_is_persistent]
+
uuid = j[:uuid].intern
provenance[uuid] = j
pips[uuid] = 0 unless pips[uuid] != nil
pipeline[component_name][:script_parameters][param_name] = param_value[:value]
elsif param_value[:default]
pipeline[component_name][:script_parameters][param_name] = param_value[:default]
- elsif param_value[:optional] != nil or param_value[:required] != nil
+ elsif param_value[:optional] != nil or param_value[:required] != nil or param_value[:dataclass] != nil
pipeline[component_name][:script_parameters][param_name] = ""
else
pipeline[component_name][:script_parameters][param_name] = param_value
provenance, pips = graph(@pipelines)
@prov_svg = ProvenanceHelper::create_provenance_graph provenance, "provenance_svg", {
+ :request => request,
:all_script_parameters => true,
:combine_jobs => :script_and_version,
:script_version_nodes => true,
@pipelines = @objects
@prov_svg = ProvenanceHelper::create_provenance_graph provenance, "provenance_svg", {
+ :request => request,
:all_script_parameters => true,
:combine_jobs => :script_and_version,
:script_version_nodes => true,
super
end
+ def index
+ @objects ||= model_class.limit(20).all
+ super
+ end
+
protected
def for_comparison v
if v.is_a? Hash or v.is_a? Array
class PipelineTemplatesController < ApplicationController
+
+ def show
+ @objects = PipelineInstance.where(pipeline_template_uuid: @object.uuid)
+ super
+ end
+
+ def show_pane_list
+ %w(Components Pipelines Attributes Metadata JSON API)
+ end
+
end
class UsersController < ApplicationController
- skip_before_filter :find_object_by_uuid, :only => :welcome
+ skip_before_filter :find_object_by_uuid, :only => [:welcome, :activity]
skip_around_filter :thread_with_mandatory_api_token, :only => :welcome
+ before_filter :ensure_current_user_is_admin, only: [:sudo, :unsetup, :setup]
def welcome
if current_user
end
end
+ def activity
+ @breadcrumb_page_name = nil
+ @users = User.limit(params[:limit] || 1000).all
+ @user_activity = {}
+ @activity = {
+ logins: {},
+ jobs: {},
+ pipeline_instances: {}
+ }
+ @total_activity = {}
+ @spans = [['This week', Time.now.beginning_of_week, Time.now],
+ ['Last week',
+ Time.now.beginning_of_week.advance(weeks:-1),
+ Time.now.beginning_of_week],
+ ['This month', Time.now.beginning_of_month, Time.now],
+ ['Last month',
+ 1.month.ago.beginning_of_month,
+ Time.now.beginning_of_month]]
+ @spans.each do |span, threshold_start, threshold_end|
+ @activity[:logins][span] = Log.
+ filter([[:event_type, '=', 'login'],
+ [:object_kind, '=', 'arvados#user'],
+ [:created_at, '>=', threshold_start],
+ [:created_at, '<', threshold_end]])
+ @activity[:jobs][span] = Job.
+ filter([[:created_at, '>=', threshold_start],
+ [:created_at, '<', threshold_end]])
+ @activity[:pipeline_instances][span] = PipelineInstance.
+ filter([[:created_at, '>=', threshold_start],
+ [:created_at, '<', threshold_end]])
+ @activity.each do |type, act|
+ records = act[span]
+ @users.each do |u|
+ @user_activity[u.uuid] ||= {}
+ @user_activity[u.uuid][span + ' ' + type.to_s] ||= 0
+ end
+ records.each do |record|
+ @user_activity[record.modified_by_user_uuid] ||= {}
+ @user_activity[record.modified_by_user_uuid][span + ' ' + type.to_s] ||= 0
+ @user_activity[record.modified_by_user_uuid][span + ' ' + type.to_s] += 1
+ @total_activity[span + ' ' + type.to_s] ||= 0
+ @total_activity[span + ' ' + type.to_s] += 1
+ end
+ end
+ end
+ @users = @users.sort_by do |a|
+ [-@user_activity[a.uuid].values.inject(:+), a.full_name]
+ end
+ # Prepend a "Total" pseudo-user to the sorted list
+ @user_activity[nil] = @total_activity
+ @users = [OpenStruct.new(uuid: nil)] + @users
+ end
+
+ def show_pane_list
+ if current_user.andand.is_admin
+ super | %w(Admin)
+ else
+ super
+ end
+ end
+
+ def index_pane_list
+ if current_user.andand.is_admin
+ super | %w(Activity)
+ else
+ super
+ end
+ end
+
+ def sudo
+ resp = $arvados_api_client.api(ApiClientAuthorization, '', {
+ api_client_authorization: {
+ owner_uuid: @object.uuid
+ }
+ })
+ redirect_to root_url(api_token: resp[:api_token])
+ end
+
def home
@showallalerts = false
@my_ssh_keys = AuthorizedKey.where(authorized_user_uuid: current_user.uuid)
- # @my_vm_perms = Link.where(tail_uuid: current_user.uuid, head_kind: 'arvados#virtual_machine', link_class: 'permission', name: 'can_login')
- # @my_repo_perms = Link.where(tail_uuid: current_user.uuid, head_kind: 'arvados#repository', link_class: 'permission', name: 'can_write')
-
@my_tag_links = {}
@my_jobs = Job.
f.html { render template: 'users/home' }
end
end
+
+ def unsetup
+ if current_user.andand.is_admin
+ @object.unsetup
+ end
+ show
+ end
+
+ def setup
+ respond_to do |format|
+ if current_user.andand.is_admin
+ setup_params = {}
+ setup_params[:send_notification_email] = "#{Rails.configuration.send_user_setup_notification_email}"
+ if params['user_uuid'] && params['user_uuid'].size>0
+ setup_params[:uuid] = params['user_uuid']
+ end
+ if params['email'] && params['email'].size>0
+ user = {email: params['email']}
+ setup_params[:user] = user
+ end
+ if params['openid_prefix'] && params['openid_prefix'].size>0
+ setup_params[:openid_prefix] = params['openid_prefix']
+ end
+ if params['repo_name'] && params['repo_name'].size>0
+ setup_params[:repo_name] = params['repo_name']
+ end
+ if params['vm_uuid'] && params['vm_uuid'].size>0
+ setup_params[:vm_uuid] = params['vm_uuid']
+ end
+
+ if User.setup setup_params
+ format.js
+ else
+ self.render_error status: 422
+ end
+ else
+ self.render_error status: 422
+ end
+ end
+ end
+
+ def setup_popup
+ @vms = VirtualMachine.all.results
+
+ @current_selections = find_current_links @object
+
+ respond_to do |format|
+ format.html
+ format.js
+ end
+ end
+
+ protected
+
+ def find_current_links user
+ current_selections = {}
+
+ if !user
+ return current_selections
+ end
+
+ # oid login perm
+ oid_login_perms = Link.where(tail_uuid: user.email,
+ head_kind: 'arvados#user',
+ link_class: 'permission',
+ name: 'can_login')
+
+ if oid_login_perms.any?
+ prefix_properties = oid_login_perms.first.properties
+ current_selections[:identity_url_prefix] = prefix_properties[:identity_url_prefix]
+ end
+
+ # repo perm
+ repo_perms = Link.where(tail_uuid: user.uuid,
+ head_kind: 'arvados#repository',
+ link_class: 'permission',
+ name: 'can_write')
+ if repo_perms.any?
+ repo_uuid = repo_perms.first.head_uuid
+ repos = Repository.where(head_uuid: repo_uuid)
+ if repos.any?
+ repo_name = repos.first.name
+ current_selections[:repo_name] = repo_name
+ end
+ end
+
+ # vm login perm
+ vm_login_perms = Link.where(tail_uuid: user.uuid,
+ head_kind: 'arvados#virtualMachine',
+ link_class: 'permission',
+ name: 'can_login')
+ if vm_login_perms.any?
+ vm_uuid = vm_login_perms.first.head_uuid
+ current_selections[:vm_uuid] = vm_uuid
+ end
+
+ return current_selections
+ end
+
end
link_name = link_uuid
if opts[:friendly_name]
- begin
- link_name = resource_class.find(link_uuid).friendly_link_name
- rescue RuntimeError
- # If that lookup failed, the link will too. So don't make one.
- return attrvalue
+ if attrvalue.respond_to? :friendly_link_name
+ link_name = attrvalue.friendly_link_name
+ else
+ begin
+ link_name = resource_class.find(link_uuid).friendly_link_name
+ rescue RuntimeError
+ # If that lookup failed, the link will too. So don't make one.
+ return attrvalue
+ end
end
end
if opts[:with_class_name]
end
def render_editable_subattribute(object, attr, subattr, template, htmloptions={})
- attrvalue = object.send(attr)
- subattr.each do |k|
- if attrvalue and attrvalue.is_a? Hash
- attrvalue = attrvalue[k]
- else
- break
+ if object
+ attrvalue = object.send(attr)
+ subattr.each do |k|
+ if attrvalue and attrvalue.is_a? Hash
+ attrvalue = attrvalue[k]
+ else
+ break
+ end
end
end
end
end
- return attrvalue if !object.attribute_editable? attr
-
- if not dataclass
- rsc = template
- if template.is_a? Hash
- if template[:value]
- rsc = template[:value]
- elsif template[:default]
- rsc = template[:default]
- end
+ rsc = template
+ if template.is_a? Hash
+ if template[:value]
+ rsc = template[:value]
+ elsif template[:default]
+ rsc = template[:default]
end
+ end
+
+ return link_to_if_arvados_object(rsc) if !object
+ return link_to_if_arvados_object(attrvalue) if !object.attribute_editable? attr
+ if dataclass
+ begin
+ dataclass = dataclass.constantize
+ rescue NameError
+ end
+ else
dataclass = ArvadosBase.resource_class_for_uuid(rsc)
end
if dataclass and dataclass.is_a? Class
items = []
- items.append({name: attrvalue, uuid: attrvalue, type: dataclass.to_s})
+ if attrvalue and !attrvalue.empty?
+ items.append({name: attrvalue, uuid: attrvalue, type: dataclass.to_s})
+ end
#dataclass.where(uuid: attrvalue).each do |item|
# items.append({name: item.uuid, uuid: item.uuid, type: dataclass.to_s})
#end
"data-placement" => "bottom",
"data-type" => datatype,
"data-url" => url_for(action: "update", id: object.uuid, controller: object.class.to_s.pluralize.underscore),
- "data-title" => "Update #{subattr[-1].to_s.titleize}",
+ "data-title" => "Set value for #{subattr[-1].to_s}",
"data-name" => dn,
"data-pk" => "{id: \"#{object.uuid}\", key: \"#{object.class.to_s.underscore}\"}",
"data-showbuttons" => "false",
:id => id
}.merge(htmloptions)
- lt += raw('<script>')
+ lt += raw("\n<script>")
if items and items.length > 0
lt += raw("add_form_selection_sources(#{items.to_json});\n")
end
def self.match(uuid)
- /^([a-f0-9]{32}(\+[0-9]+)?)(\+.*)?$/.match(uuid.to_s)
+ /^([a-f0-9]{32})(\+[0-9]+)?(\+.*?)?(\/.*)?$/.match(uuid.to_s)
end
end
module PipelineInstancesHelper
- def pipeline_summary object=nil
- object ||= @object
- ret = {todo:0, running:0, queued:0, done:0, failed:0, total:0}
- object.components.values.each do |c|
- ret[:total] += 1
- case
- when !c[:job]
- ret[:todo] += 1
- when c[:job][:success]
- ret[:done] += 1
- when c[:job][:failed]
- ret[:failed] += 1
- when c[:job][:finished_at]
- ret[:running] += 1 # XXX finished but !success and !failed??
- when c[:job][:started_at]
- ret[:running] += 1
- else
- ret[:queued] += 1
- end
- end
- ret.merge! Hash[ret.collect do |k,v|
- [('percent_' + k.to_s).to_sym,
- ret[:total]<1 ? 0 : (100.0*v/ret[:total]).floor]
- end]
- ret
- end
def pipeline_jobs object=nil
object ||= @object
end
def render_pipeline_job pj
- if pj[:percent_done]
- pj[:progress_bar] = raw <<EOF
-<div class="progress" style="width:100px">
- <span class="progress-bar progress-bar-success" style="width:#{pj[:percent_done]}%"></span>
- <span class="progress-bar" style="width:#{pj[:percent_running]}%"></span>
-</div>
-EOF
- elsif pj[:progress]
- raw <<EOF
-<div class="progress" style="width:100px">
-<span class="progress-bar" style="width:#{pj[:progress]*100}%">
-</span>
-</div>
-EOF
- end
+ pj[:progress_bar] = render partial: 'job_progress', locals: {:j => pj[:job]}
pj[:output_link] = link_to_if_arvados_object pj[:output]
pj[:job_link] = link_to_if_arvados_object pj[:job][:uuid]
pj
end
+
protected
def pipeline_jobs_newschool object
ret = []
i = -1
- object.components.each do |cname, c|
+
+ comp = []
+
+ template = PipelineTemplate.find(@object.pipeline_template_uuid) rescue nil
+ if template
+ order = PipelineTemplatesHelper::sort_components(template.components)
+ order.each do |k|
+ if object.components[k]
+ comp.push([k, object.components[k]])
+ end
+ end
+ else
+ object.components.each do |k, v|
+ comp.push([k, v])
+ end
+ end
+
+ comp.each do |cname, c|
+ puts cname, c
i += 1
pj = {index: i, name: cname}
pj[:job] = c[:job].is_a?(Hash) ? c[:job] : {}
+require 'tsort'
+
+class Hash
+ include TSort
+ def tsort_each_node(&block)
+ keys.sort.each(&block)
+ end
+
+ def tsort_each_child(node)
+ if self[node]
+ self[node][:script_parameters].sort.map do |k, v|
+ if v.is_a? Hash and v[:output_of]
+ yield v[:output_of].to_sym
+ end
+ end
+ end
+ end
+end
+
module PipelineTemplatesHelper
+ def self.sort_components(components)
+ components.tsort
+ end
end
@opts = opts
@visited = {}
@jobs = {}
+ @node_extra = {}
end
-
+
def self.collection_uuid(uuid)
m = CollectionsHelper.match(uuid)
if m
- #if m[2]
- return m[1]
- #else
- # Collection.where(uuid: ['contains', m[1]]).each do |u|
- # puts "fixup #{uuid} to #{u.uuid}"
- # return u.uuid
- # end
- #end
+ if m[2]
+ return m[1]+m[2]
+ else
+ return m[1]
+ end
else
nil
end
end
+ def url_for u
+ p = { :host => @opts[:request].host,
+ :port => @opts[:request].port,
+ :protocol => @opts[:request].protocol }
+ p.merge! u
+ Rails.application.routes.url_helpers.url_for (p)
+ end
+
def determine_fillcolor(n)
fillcolor = %w(aaaaaa aaffaa aaaaff aaaaaa ffaaaa)[n || 0] || 'aaaaaa'
"style=filled,fillcolor=\"##{fillcolor}\""
end
def describe_node(uuid)
- bgcolor = determine_fillcolor @opts[:pips][uuid] if @opts[:pips]
+ uuid = uuid.to_sym
+ bgcolor = determine_fillcolor @opts[:pips].andand[uuid]
rsc = ArvadosBase::resource_class_for_uuid uuid.to_s
if rsc
- href = "/#{rsc.to_s.underscore.pluralize rsc}/#{uuid}"
+ href = url_for ({:controller => rsc.to_s.tableize,
+ :action => :show,
+ :id => uuid.to_s })
#"\"#{uuid}\" [label=\"#{rsc}\\n#{uuid}\",href=\"#{href}\"];\n"
if rsc == Collection
- #puts uuid
- if uuid == :"d41d8cd98f00b204e9800998ecf8427e+0"
+ if Collection.is_empty_blob_locator? uuid.to_s
# special case
- #puts "empty!"
return "\"#{uuid}\" [label=\"(empty collection)\"];\n"
end
if @pdata[uuid]
- #puts @pdata[uuid]
if @pdata[uuid][:name]
return "\"#{uuid}\" [label=\"#{@pdata[uuid][:name]}\",href=\"#{href}\",shape=oval,#{bgcolor}];\n"
- else
+ else
files = nil
if @pdata[uuid].respond_to? :files
files = @pdata[uuid].files
if i < files.length
label += "\\n⋮"
end
- return "\"#{uuid}\" [label=\"#{label}\",href=\"#{href}\",shape=oval,#{bgcolor}];\n"
+ extra_s = @node_extra[uuid].andand.map { |k,v|
+ "#{k}=\"#{v}\""
+ }.andand.join ","
+ return "\"#{uuid}\" [label=\"#{label}\",href=\"#{href}\",shape=oval,#{bgcolor},#{extra_s}];\n"
end
end
end
- return "\"#{uuid}\" [label=\"#{rsc}\",href=\"#{href}\",#{bgcolor}];\n"
end
+ return "\"#{uuid}\" [label=\"#{rsc}\",href=\"#{href}\",#{bgcolor}];\n"
end
"\"#{uuid}\" [#{bgcolor}];\n"
end
gr = "\"#{head}\" -> \"#{tail}\""
end
if extra.length > 0
- gr += "["
+ gr += " ["
extra.each do |k, v|
gr += "#{k}=\"#{v}\","
end
end
unless node == ""
node += "']"
- #puts node
- #id = "#{job[:uuid]}_#{prefix}"
gr += "\"#{node}\" [label=\"#{node}\"];\n"
gr += edge(job_uuid(job), node, {:label => prefix})
end
when String
return '' if sp.empty?
m = GenerateGraph::collection_uuid(sp)
- #puts "#{m} pdata is #{@pdata[m.intern]}"
if m and (@pdata[m.intern] or (not @opts[:pdata_only]))
gr += edge(job_uuid(job), m, {:label => prefix})
gr += generate_provenance_edges(m)
elsif @opts[:all_script_parameters]
- #id = "#{job[:uuid]}_#{prefix}"
gr += "\"#{sp}\" [label=\"#{sp}\"];\n"
gr += edge(job_uuid(job), sp, {:label => prefix})
end
uuid = uuid.intern if uuid
if (not uuid) or uuid.empty? or @visited[uuid]
-
- #puts "already @visited #{uuid}"
return ""
end
@visited[uuid] = true
end
- #puts "visiting #{uuid}"
-
- if m
+ if m
# uuid is a collection
- gr += describe_node(uuid)
-
- if m == :"d41d8cd98f00b204e9800998ecf8427e+0"
- # empty collection, don't follow any further
- return gr
- end
-
- @pdata.each do |k, job|
- if job[:output] == uuid.to_s
- gr += edge(uuid, job_uuid(job), {:label => "output"})
- gr += generate_provenance_edges(job[:uuid])
- end
- if job[:log] == uuid.to_s
- gr += edge(uuid, job_uuid(job), {:label => "log"})
- gr += generate_provenance_edges(job[:uuid])
+ if not Collection.is_empty_blob_locator? uuid.to_s
+ @pdata.each do |k, job|
+ if job[:output] == uuid.to_s
+ extra = { label: 'output' }
+ if job[:output_is_persistent]
+ extra[:label] += ' (persistent)'
+ @node_extra[uuid] ||= {}
+ @node_extra[uuid][:penwidth] = 4
+ end
+ gr += edge(uuid, job_uuid(job), extra)
+ gr += generate_provenance_edges(job[:uuid])
+ end
+ if job[:log] == uuid.to_s
+ gr += edge(uuid, job_uuid(job), {:label => "log"})
+ gr += generate_provenance_edges(job[:uuid])
+ end
end
end
+ gr += describe_node(uuid)
else
# uuid is something else
rsc = ArvadosBase::resource_class_for_uuid uuid.to_s
gr += edge(job_uuid(job), job[:script_version], {:label => "script_version"})
end
end
+ elsif rsc == Link
+ # do nothing
else
gr += describe_node(uuid)
end
@pdata.each do |k, link|
if link[:head_uuid] == uuid.to_s and link[:link_class] == "provenance"
+ href = url_for ({:controller => Link.to_s.tableize,
+ :action => :show,
+ :id => link[:uuid] })
+
gr += describe_node(link[:tail_uuid])
- gr += edge(link[:head_uuid], link[:tail_uuid], {:label => link[:name], :href => "/links/#{link[:uuid]}"})
+ gr += edge(link[:head_uuid], link[:tail_uuid], {:label => link[:name], :href => href})
gr += generate_provenance_edges(link[:tail_uuid])
end
end
- #puts "finished #{uuid}"
-
gr
end
def describe_jobs
gr = ""
@jobs.each do |k, v|
- gr += "\"#{k}\" [href=\"/jobs?"
+ href = url_for ({:controller => Job.to_s.tableize,
+ :action => :index })
+
+ gr += "\"#{k}\" [href=\"#{href}?"
n = 0
v.each do |u|
gr += "\",label=\""
if @opts[:combine_jobs] == :script_only
- gr += uuid = "#{v[0][:script]}"
+ gr += "#{v[0][:script]}"
elsif @opts[:combine_jobs] == :script_and_version
- gr += uuid = "#{v[0][:script]}"
+ gr += "#{v[0][:script]}" # Just show the name but the nodes will be distinct
else
- gr += uuid = "#{v[0][:script]}\\n#{v[0][:finished_at]}"
+ gr += "#{v[0][:script]}\\n#{v[0][:finished_at]}"
end
gr += "\",#{determine_fillcolor n}];\n"
end
gr += "edge [dir=back];"
end
- #puts "@pdata is #{pdata}"
-
g = GenerateGraph.new(pdata, opts)
pdata.each do |k, v|
gr += "}"
svg = ""
- #puts gr
-
require 'open3'
Open3.popen2("dot", "-Tsvg") do |stdin, stdout, wait_thr|
@@client_mtx = Mutex.new
@@api_client = nil
- @@profiling_enabled = Rails.configuration.profiling_enabled rescue false
+ @@profiling_enabled = Rails.configuration.profiling_enabled
def api(resources_kind, action, data=nil)
profile_checkpoint
resp
end
+ def self.patch_paging_vars(ary, items_available, offset, limit)
+ if items_available
+ (class << ary; self; end).class_eval { attr_accessor :items_available }
+ ary.items_available = items_available
+ end
+ if offset
+ (class << ary; self; end).class_eval { attr_accessor :offset }
+ ary.offset = offset
+ end
+ if limit
+ (class << ary; self; end).class_eval { attr_accessor :limit }
+ ary.limit = limit
+ end
+ ary
+ end
+
def unpack_api_response(j, kind=nil)
if j.is_a? Hash and j[:items].is_a? Array and j[:kind].match(/(_list|List)$/)
ary = j[:items].collect { |x| unpack_api_response x, j[:kind] }
- if j[:items_available]
- (class << ary; self; end).class_eval { attr_accessor :items_available }
- ary.items_available = j[:items_available]
- end
- ary
+ ArvadosApiClient::patch_paging_vars(ary, j[:items_available], j[:offset], j[:limit])
elsif j.is_a? Hash and (kind || j[:kind])
oclass = self.kind_class(kind || j[:kind])
if oclass
Rails.configuration.arvados_v1_base
end
- def arvados_schema
- @arvados_schema ||= api 'schema', ''
- end
-
def discovery
@discovery ||= api '../../discovery/v1/apis/arvados/v1/rest', ''
end
'modified_by_user_uuid' => '004',
'modified_by_client_uuid' => '005',
'name' => '050',
- 'tail_kind' => '100',
'tail_uuid' => '100',
- 'head_kind' => '101',
'head_uuid' => '101',
'info' => 'zzz-000',
'updated_at' => 'zzz-999'
return @columns unless @columns.nil?
@columns = []
@attribute_info ||= {}
- return @columns if $arvados_api_client.arvados_schema[self.to_s.to_sym].nil?
- $arvados_api_client.arvados_schema[self.to_s.to_sym].each do |coldef|
- k = coldef[:name].to_sym
- if coldef[:type] == coldef[:type].downcase
- @columns << column(k, coldef[:type].to_sym)
+ schema = $arvados_api_client.discovery[:schemas][self.to_s.to_sym]
+ return @columns if schema.nil?
+ schema[:properties].each do |k, coldef|
+ case k
+ when :etag, :kind
+ attr_reader k
else
- @columns << column(k, :text)
- serialize k, coldef[:type].constantize
+ if coldef[:type] == coldef[:type].downcase
+ # boolean, integer, etc.
+ @columns << column(k, coldef[:type].to_sym)
+ else
+ # Hash, Array
+ @columns << column(k, :text)
+ serialize k, coldef[:type].constantize
+ end
+ attr_accessible k
+ @attribute_info[k] = coldef
end
- attr_accessible k
- @attribute_info[k] = coldef
end
- attr_reader :etag
- attr_reader :kind
@columns
end
ArvadosResourceList.new(self).order(*args)
end
+ def self.filter(*args)
+ ArvadosResourceList.new(self).filter(*args)
+ end
+
def self.where(*args)
ArvadosResourceList.new(self).where(*args)
end
true
end
end
-
+
def links(*args)
o = {}
o.merge!(args.pop) if args[-1].is_a? Hash
o[:link_class] ||= args.shift
o[:name] ||= args.shift
- o[:head_kind] ||= args.shift
- o[:tail_kind] = self.kind
o[:tail_uuid] = self.uuid
if all_links
return all_links.select do |m|
self
end
+ def offset(skip)
+ @offset = skip
+ self
+ end
+
def order(orderby_spec)
@orderby_spec = orderby_spec
self
end
+ def filter _filters
+ @filters ||= []
+ @filters += _filters
+ self
+ end
+
def where(cond)
cond = cond.dup
cond.keys.each do |uuid_key|
}
api_params[:eager] = '1' if @eager
api_params[:limit] = @limit if @limit
+ api_params[:offset] = @offset if @offset
api_params[:order] = @orderby_spec if @orderby_spec
+ api_params[:filters] = @filters if @filters
res = $arvados_api_client.api @resource_class, '', api_params
@results = $arvados_api_client.unpack_api_response res
self
@results
end
+ def results=(r)
+ @results = r
+ end
+
def all
where({})
end
def items_available
results.items_available if results.respond_to? :items_available
end
+
+ def result_limit
+ results.limit if results.respond_to? :limit
+ end
+
+ def result_offset
+ results.offset if results.respond_to? :offset
+ end
+
end
class Collection < ArvadosBase
+
+ MD5_EMPTY = 'd41d8cd98f00b204e9800998ecf8427e'
+
+ # Return true if the given string is the locator of a zero-length blob
+ def self.is_empty_blob_locator? locator
+ !!locator.to_s.match("^#{MD5_EMPTY}(\\+.*)?\$")
+ end
+
def total_bytes
if files
tot = 0
$arvados_api_client.api "collections/#{self.uuid}/", "used_by"
end
- # def selection_label
- # name = ''
- # i = 0
- # if self.files.length > 3
- # m = 3
- # else
- # m = self.files.length
- # end
- # while i < m
- # name += "#{self.files[i][1]}"
- # i += 1
- # name += ", " if i < m
- # end
- # if i < self.files.length
- # name += "&ellip;"
- # end
- # name
- # end
end
attr_accessor :head
attr_accessor :tail
def self.by_tail(t, opts={})
- where(opts.merge :tail_kind => t.kind, :tail_uuid => t.uuid)
+ where(opts.merge :tail_uuid => t.uuid)
end
end
def friendly_link_name
[self.first_name, self.last_name].compact.join ' '
end
+
+ def unsetup
+ self.private_reload($arvados_api_client.api(self.class,
+ "/#{self.uuid}/unsetup",
+ {}))
+ end
+
+ def self.setup params
+ $arvados_api_client.api(self, "/setup", params)
+ end
+
end
<% end %>
<% content_for :js do %>
- $(window).on('load', function() {
- $('ul.nav-tabs > li > a').on('shown.bs.tab', smart_scroll_fixup);
- });
+ $(window).on('load', smart_scroll_fixup);
+ $(document).on('shown.bs.tab', 'ul.nav-tabs > li > a', smart_scroll_fixup);
<% end %>
--- /dev/null
+<% percent_total_tasks = 100 / (j[:tasks_summary][:done] + j[:tasks_summary][:running] + j[:tasks_summary][:failed] + j[:tasks_summary][:todo]) rescue 0 %>
+
+<% if defined? scaleby %>
+ <% percent_total_tasks *= scaleby %>
+<% end %>
+
+<% if not defined? scaleby %>
+ <div class="progress">
+<% end %>
+
+<span class="progress-bar progress-bar-success" style="width: <%= j[:tasks_summary][:done] * percent_total_tasks rescue 0 %>%;">
+</span>
+<span class="progress-bar progress-bar-danger" style="width: <%= j[:tasks_summary][:failed] * percent_total_tasks rescue 0 %>%;">
+</span>
+<span class="progress-bar" style="width: <%= j[:tasks_summary][:running] * percent_total_tasks rescue 0 %>%;">
+</span>
+
+<% if not defined? scaleby %>
+</div>
+<% end %>
--- /dev/null
+<% if j[:success] %>
+ <span class="label label-success"><%= if defined? title then title else 'success' end %></span>
+<% elsif j[:success] == false %>
+ <span class="label label-danger"><%= if defined? title then title else 'failed' end %></span>
+<% elsif j[:finished_at] %>
+ <span class="label label-default"><%= if defined? title then title else 'finished' end %></span>
+<% elsif j[:started_at] %>
+ <span class="label label-info"><%= if defined? title then title else 'running' end %></span>
+<% else %>
+ <span class="label label-default"><%= if defined? title then title else 'not running' end %></span>
+<% end %>
--- /dev/null
+<% content_for :css do %>
+.index-paging {
+text-align: center;
+padding-left: 1em;
+padding-right: 1em;
+background-color: whitesmoke;
+}
+.paging-number {
+display: inline-block;
+min-width: 1.2em;
+}
+<% end %>
+
+<% if results.respond_to? :result_offset and
+ results.respond_to? :result_limit and
+ results.respond_to? :items_available and
+ results.result_offset != nil and
+ results.result_limit != nil and
+ results.items_available != nil
+%>
+<div class="index-paging">
+ Displaying <%= results.result_offset+1 %> –
+ <%= if results.result_offset + results.result_limit > results.items_available
+ results.items_available
+ else
+ results.result_offset + results.result_limit
+ end %>
+ out of <%= results.items_available %>
+</div>
+
+<% if not (results.result_offset == 0 and results.items_available <= results.result_limit) %>
+
+<div class="index-paging">
+
+<% if results.result_offset > 0 %>
+ <% if results.result_offset > results.result_limit %>
+ <% prev_offset = results.result_offset - results.result_limit %>
+ <% else %>
+ <% prev_offset = 0 %>
+ <% end %>
+<% else %>
+ <% prev_offset = nil %>
+<% end %>
+
+<% this_offset = results.result_offset %>
+
+<% if (results.result_offset + results.result_limit) < results.items_available %>
+ <% next_offset = results.result_offset + results.result_limit %>
+<% else %>
+ <% next_offset = nil %>
+<% end %>
+
+<span class="pull-left">
+<% if results.result_offset > 0 %>
+ <%= link_to raw("<span class='glyphicon glyphicon-fast-backward'></span>"), {:id => object, :offset => 0, :limit => results.result_limit} %>
+<% else %>
+ <span class='glyphicon glyphicon-fast-backward text-muted'></span>
+<% end %>
+
+<% if prev_offset %>
+ <%= link_to raw("<span class='glyphicon glyphicon-step-backward'></span>"), {:id => object, :offset => prev_offset, :limit => results.result_limit} %>
+<% else %>
+<span class='glyphicon glyphicon-step-backward text-muted'></span>
+<% end %>
+</span>
+
+<% first = this_offset - (10 * results.result_limit) %>
+<% last = this_offset + (11 * results.result_limit) %>
+
+<% lastpage_offset = (results.items_available / results.result_limit) * results.result_limit %>
+
+<% if last > results.items_available %>
+ <% first -= (last - lastpage_offset) %>
+ <% last -= (last - results.items_available) %>
+<% end %>
+
+<% if first < 0 %>
+ <% d = -first %>
+ <% first += d %>
+ <% last += d %>
+<% end %>
+
+<% last = results.items_available if last > results.items_available %>
+
+<% i = first %>
+<% n = first / results.result_limit %>
+
+<% if first > 0 %>
+…
+<% end %>
+
+<% while i < last %>
+<% if i != this_offset %>
+ <%= link_to "#{n+1}", {:id => @object, :offset => i, :limit => results.result_limit}, class: 'paging-number' %>
+<% else %>
+ <span class="paging-number" style="font-weight: bold;"><%= n+1 %></span>
+<% end %>
+<% i += results.result_limit %>
+<% n += 1 %>
+<% end %>
+
+<% if last < results.items_available %>
+…
+<% end %>
+
+<span class="pull-right">
+<% if next_offset %>
+ <%= link_to raw("<span class='glyphicon glyphicon-step-forward'></span>"), {:id => @object, :offset => next_offset, :limit => results.result_limit} %>
+<% else %>
+<span class='glyphicon glyphicon-forward text-muted'></span>
+<% end %>
+
+<% if (results.items_available - results.result_offset) >= results.result_limit %>
+ <%= link_to raw("<span class='glyphicon glyphicon-fast-forward'></span>"), {:id => @object, :offset => results.items_available - (results.items_available % results.result_limit),
+ :limit => results.result_limit} %>
+<% else %>
+ <span class='glyphicon glyphicon-fast-forward text-muted'></span>
+<% end %>
+
+</span>
+
+</div>
+
+<% end %>
+
+<% end %>
--- /dev/null
+<% component_frac = 1.0 / p.components.length %>
+<div class="progress">
+ <% p.components.each do |k,c| %>
+ <% if c[:job] %>
+ <%= render partial: "job_progress", locals: {:j => c[:job], :scaleby => component_frac } %>
+ <% end %>
+ <% end %>
+</div>
--- /dev/null
+<% if p.success %>
+ <span class="label label-success">finished</span>
+<% elsif p.success == false %>
+ <span class="label label-danger">failed</span>
+<% elsif p.active %>
+ <span class="label label-info">running</span>
+<% else %>
+ <% if (p.components.select do |k,v| v[:job] end).length == 0 %>
+ <span class="label label-default">not started</span>
+ <% else %>
+ <span class="label label-default">not running</span>
+ <% end %>
+<% end %>
<% attr_blacklist = ' created_at modified_at modified_by_user_uuid modified_by_client_uuid updated_at' %>
+<%= render partial: "paging", locals: {results: @objects, object: @object} %>
+
<%= form_tag do |f| %>
<table class="table table-condensed arv-index">
<% end %>
+<%= render partial: "paging", locals: {results: @objects, object: @object} %>
+
<% end %>
<% content_for :tab_line_buttons do %>
-<% if controller.model_class.creatable? %>
-<%= button_to "Add a new #{controller.model_class.to_s.underscore.gsub '_', ' '}",
- { action: 'create', return_to: request.url },
- { class: 'btn btn-primary pull-right' } %>
-<% end %>
+ <% if controller.model_class.creatable? %>
+
+ <% if controller.model_class.name == 'User' %>
+ <%= link_to "Add a new #{controller.model_class.to_s.underscore.gsub '_', ' '}", setup_user_popup_path,
+ {class: 'btn btn-primary pull-right', :remote => true, 'data-toggle' => "modal",
+ 'data-target' => '#user-setup-modal-window', return_to: request.url} %>
+ <div id="user-setup-modal-window" class="modal fade" role="dialog" aria-labelledby="myModalLabel" aria-hidden="true"></div>
+ <% else %>
+ <%= button_to "Add a new #{controller.model_class.to_s.underscore.gsub '_', ' '}",
+ { action: 'create', return_to: request.url },
+ { class: 'btn btn-primary pull-right' } %>
+ <% end %>
+
+ <% end %>
<% end %>
<% content_for :css do %>
.file-list-inline-image {
width: 50%;
- height: auto;
+ height: auto;
}
<% end %>
</tr>
</thead><tbody>
<% if @object then @object.files.sort_by{|f|[f[0],f[1]]}.each do |file| %>
- <% file_path = "#{file[0]}/#{file[1]}" %>
- <tr>
- <td>
- <% fp2 = file_path[2..-1] if file_path[0..1] == './' %>
- <% fp2 ||= file_path %>
-<%= check_box_tag 'uuids[]', @object.uuid+file_path, false, {
- :class => 'persistent-selection',
- :friendly_type => "File",
- :friendly_name => "#{@object.uuid}/#{fp2}",
- :href => "#{url_for controller: 'collections', action: 'show', id: @object.uuid }/#{file_path}"
- } %>
- </td>
- <td>
- <%= file[0] %>
- </td>
+ <% f0 = file[0] %>
+ <% f0 = '' if f0 == '.' %>
+ <% f0 = f0[2..-1] if f0[0..1] == './' %>
+ <% f0 += '/' if not f0.empty? %>
+ <% file_path = "#{f0}#{file[1]}" %>
+ <tr>
+ <td>
+ <%= check_box_tag 'uuids[]', @object.uuid+'/'+file_path, false, {
+ :class => 'persistent-selection',
+ :friendly_type => "File",
+ :friendly_name => "#{@object.uuid}/#{file_path}",
+ :href => "#{url_for controller: 'collections', action: 'show', id: @object.uuid }/#{file_path}"
+ } %>
+ </td>
+ <td>
+ <%= file[0] %>
+ </td>
<td>
<%= link_to((if /\.(jpg|jpeg|gif|png|svg)$/i.match(file[1]) then
image_tag "#{url_for @object}/#{file_path}", class: "file-list-inline-image"
else
file[1]
- end),
- {controller: 'collections', action: 'show_file', uuid: @object.uuid, file: file_path, size: file[2], disposition: 'inline'},
+ end),
+ {controller: 'collections', action: 'show_file', uuid: @object.uuid, file: file_path, size: file[2], disposition: 'inline'},
{title: file_path}) %>
</td>
- <td style="text-align:right">
- <%= raw(human_readable_bytes_html(file[2])) %>
- </td>
+ <td style="text-align:right">
+ <%= raw(human_readable_bytes_html(file[2])) %>
+ </td>
- <td>
- <div style="display:inline-block">
- <%= link_to raw('<i class="glyphicon glyphicon-download-alt"></i>'), {controller: 'collections', action: 'show_file', uuid: @object.uuid, file: file_path, size: file[2], disposition: 'attachment'}, {class: 'btn btn-info btn-sm', title: 'Download'} %>
- </div>
- </td>
- </tr>
+ <td>
+
<div style="display:inline-block">
+ <%= link_to raw('<i class="glyphicon glyphicon-download-alt"></i>'), {controller: 'collections', action: 'show_file', uuid: @object.uuid, file: file_path, size: file[2], disposition: 'attachment'}, {class: 'btn btn-info btn-sm', title: 'Download'} %>
+ </div>
+ </td>
+ </tr>
<% end; end %>
</tbody>
</table>
</div>
<% end %>
+<%= render partial: "paging", locals: {results: @collections, object: @object} %>
+
<div style="padding-right: 1em">
<%= form_tag do |f| %>
-<table id="collections-index" class="topalign table table-condensed table-fixedlayout table-fixed-header-row">
+<table id="collections-index" class="topalign table table-condensed table-fixedlayout"> <!-- table-fixed-header-row -->
<colgroup>
<col width="4%" />
<col width="10%" />
</div>
+<%= render partial: "paging", locals: {results: @collections, object: @object} %>
+
<% content_for :footer_js do %>
$(document).on('click', 'form[data-remote] input[type=submit]', function() {
$('table#collections-index tbody').fadeTo(200, 0.3);
+<%= render partial: "paging", locals: {results: @groups, object: @object} %>
+
<table class="table table-hover">
<thead>
<tr class="contain-align-left">
</tbody>
</table>
+
+<%= render partial: "paging", locals: {results: @groups, object: @object} %>
<i class="icon-plus-sign expand-collapse-row" data-id="<%= j.uuid %>" style="cursor: pointer"></i>
</td>
<td>
- <% if j.success == false %>
- <span class="badge badge-warning" title="fail">✖</span>
- <% elsif j.success %>
- <span class="badge badge-success" title="success">✔</span>
- <% elsif j.running %>
- <span class="badge badge-info" title="running">✈</span>
- <% else %>
- <span class="badge" title="queued">✉</span>
- <% end %>
+ <%= render partial: 'job_status_label', locals: {:j => j} %>
</td>
<td>
- <% if j.started_at and not j.finished_at %>
- <% percent_total_tasks = 100 / (j.tasks_summary[:running] + j.tasks_summary[:done] + j.tasks_summary[:todo]) rescue 0 %>
- <div class="progress" style="margin-bottom: 0">
- <div class="bar bar-success" style="width: <%= j.tasks_summary[:done] * percent_total_tasks rescue 0 %>%;"></div>
- <div class="bar" style="width: <%= j.tasks_summary[:running] * percent_total_tasks rescue 0 %>%; opacity: 0.3"></div>
+ <div class="inline-progress-container">
+ <%= render partial: 'job_progress', locals: {:j => j} %>
</div>
- <% end %>
</td>
<td>
<%= link_to_if_arvados_object j.uuid %>
-->
<li class="dropdown notification-menu">
- <a href="#" class="dropdown-toggle" data-toggle="dropdown">
+ <a href="#" class="dropdown-toggle" data-toggle="dropdown" id="collections-menu">
<span class="glyphicon glyphicon-paperclip"></span>
<span class="badge" id="persistent-selection-count"></span>
<span class="caret"></span>
</a>
- <ul class="dropdown-menu" role="menu" id="persistent-selection-list">
+ <ul class="dropdown-menu" role="menu" id="persistent-selection-list">
+ <%= form_tag '/actions' do %>
+ <div id="selection-form-content"></div>
+ <% end %>
</ul>
</li>
<% if current_user.is_active %>
<li class="dropdown notification-menu">
- <a href="#" class="dropdown-toggle" data-toggle="dropdown">
+ <a href="#" class="dropdown-toggle" data-toggle="dropdown" id="notifications-menu">
<span class="glyphicon glyphicon-envelope"></span>
<span class="badge badge-alert notification-count"><%= @notification_count %></span>
<span class="caret"></span>
<% end %>
<li class="dropdown">
- <a href="#" class="dropdown-toggle" data-toggle="dropdown">
+ <a href="#" class="dropdown-toggle" data-toggle="dropdown" id="user-menu">
<span class="glyphicon glyphicon-user"></span><span class="caret"></span>
</a>
<ul class="dropdown-menu" role="menu">
</div>
</div>
+ <%= yield :footer_html %>
<%= piwik_tracking_tag %>
<%= javascript_tag do %>
<%= yield :footer_js %>
<p><%= image_tag "dax.png", class: "dax" %>
Hi, I noticed you haven't run a pipeline yet.
<%= link_to "Click here to learn how to run an Arvados Crunch pipeline.",
- "#{Rails.configuration.arvados_docsite}/user/tutorials/tutorial-new-pipeline.html",
+ "#{Rails.configuration.arvados_docsite}/user/tutorials/tutorial-pipeline-workbench.html",
style: "font-weight: bold",
target: "_blank" %>
</p>
<% content_for :css do %>
- .pipeline_color_legend {
- padding-left: 1em;
- padding-right: 1em;
- }
-table.pipeline-components-table {
- width: 100%;
- table-layout: fixed;
- overflow: hidden;
-}
-
-table.pipeline-components-table thead th {
- text-align: bottom;
-}
-table.pipeline-components-table div.progress {
- margin-bottom: 0;
-}
-
-table.pipeline-components-table td {
- overflow: hidden;
- text-overflow: ellipsis;
-}
-
-td.required {
- background: #ffdddd;
-}
<% end %>
-<% template = PipelineTemplate.find(@object.pipeline_template_uuid) %>
-<% if template %>
- <h2><%= template.name %></h2>
+<% template = PipelineTemplate.find(@object.pipeline_template_uuid) rescue nil %>
+
+<%= content_for :content_top do %>
+ <h2>
+ <%= render_editable_attribute @object, 'name', nil, { 'data-emptytext' => 'Unnamed pipeline', 'data-mode' => 'inline' } %>
+ </h2>
+ <% if template %>
+ <h4>
+ From template:
+ <%= link_to_if_arvados_object template, friendly_name: true %>
+ </h4>
+ <% end %>
<% end %>
<% if @object.active != nil %>
<col style="width: 15%" />
<col style="width: 20%" />
<col style="width: 12%" />
- <col style="width: 8%" />
+ <col style="width: 12%" />
<col style="width: 45%" />
</colgroup>
<thead>
<% render_pipeline_jobs.each do |pj| %>
<tr>
<td>
- <% label = if pj[:job].andand[:uuid]
- if pj[:job][:success] == true
- 'label-success'
- elsif pj[:job][:success] == false
- 'label-danger'
- elsif pj[:job][:running] == true
- 'label-info'
- else
- 'label-default'
- end
- else
- 'label-default'
- end %>
+ <% job_status = render(partial: 'job_status_label',
+ locals: { :j => pj[:job], :title => pj[:name] }) %>
<% if pj[:job].andand[:uuid] %>
- <%= link_to pj[:name], job_url(id: pj[:job][:uuid]), class: "label #{label}" %>
+ <%= link_to(job_status, job_url(id: pj[:job][:uuid])) %>
<% else %>
- <span class="label <%= label %>"><%= pj[:name] %></span>
+ <%= job_status %>
<% end %>
-
</td><td>
<%= pj[:script] %>
<br /><span class="deemphasize"><%= pj[:script_version] %></span>
</td><td>
<%= pj[:progress_bar] %>
</td><td>
- <% if pj[:job].andand[:cancelled_at] %>
- <span class="label label-warning">cancelled</span>
- <% elsif pj[:failed] %>
- <span class="label label-danger">failed</span>
- <% elsif pj[:result] == 'queued' %>
- <span class="label">queued</span>
- <% end %>
+ <%= render(partial: 'job_status_label',
+ locals: { :j => pj[:job] }) %>
</td><td>
<%= link_to_if_arvados_object pj[:output] %>
</td>
<% end %>
</tbody>
<tfoot>
- <tr><td colspan="4"></td></tr>
+ <tr><td colspan="5"></td></tr>
</tfoot>
</table>
<% content_for :js do %>
setInterval(function(){$('a.refresh').click()}, 15000);
<% end %>
-<% end %>
-
-<% else %>
+<% content_for :tab_line_buttons do %>
<%= form_tag @object, :method => :put do |f| %>
- <%= hidden_field @object.class.to_s.underscore.singularize.to_sym, :active, :value => true %>
- <%= button_tag "Run pipeline", {class: 'btn btn-primary pull-right', id: "run-pipeline-button"} %>
- <% end %>
+ <%= hidden_field @object.class.to_s.underscore.singularize.to_sym, :active, :value => false %>
-<table class="table pipeline-components-table" style="margin-top: -.1em">
- <colgroup>
- <col style="width: 15%" />
- <col style="width: 20%" />
- <col style="width: 20%" />
- <col style="width: 45%" />
- </colgroup>
+ <%= button_tag "Stop pipeline", {class: 'btn btn-primary pull-right', id: "run-pipeline-button"} %>
+ <% end %>
+<% end %>
- <thead>
- <tr>
- <th>
- component
- </th><th>
- script
- </th><th>
- parameter
- </th><th>
- value
- </th>
- </tr>
- </thead>
- <tbody>
- <% template.components.each do |k, template_value| %>
+<% end %>
- <tr>
- <td><span class="label label-default"><%= k %></span></td>
+<% else %>
- <td><%= render_editable_subattribute @object, :components, [k, :script], template_value[:script] %></td>
+ <p>Please set the desired input parameters for the components of this pipeline. Parameters highlighted in red are required.</p>
- <td>script version</td>
+ <% content_for :tab_line_buttons do %>
+ <%= form_tag @object, :method => :put do |f| %>
- <td>
- <%= render_editable_subattribute @object, :components, [k, :script_version], template_value[:script_version] %>
- </td>
- </tr>
+ <%= hidden_field @object.class.to_s.underscore.singularize.to_sym, :active, :value => true %>
- <% if template_value[:script_parameters].length > 0 %>
- <% template_value[:script_parameters].each do |p, tv| %>
- <tr>
- <td style="border-top: none"></td>
- <td style="border-top: none"></td>
-
- <td class="property-edit-row"><%= p %></td>
- <td class="property-edit-row"><%= render_editable_subattribute @object, :components, [k, :script_parameters, p.to_sym], tv %></td>
- <% end %>
- </tr>
+ <%= button_tag "Run pipeline", {class: 'btn btn-primary pull-right', id: "run-pipeline-button"} %>
<% end %>
<% end %>
- </tbody>
- </table>
-
+
+ <%= render partial: 'pipeline_templates/show_components_template', locals: {:template => template, :obj => @object} %>
+
<% end %>
<% end rescue nil %>
<% end %>
+<%= render partial: "paging", locals: {results: @objects, object: @object} %>
+
<%= form_tag do |f| %>
<table class="table table-condensed table-fixedlayout">
<colgroup>
<col width="5%" />
- <col width="10%" />
+ <col width="15%" />
+ <col width="25%" />
<col width="20%" />
- <col width="10%" />
- <col width="30%" />
<col width="15%" />
- <col width="10%" />
+ <col width="20%" />
</colgroup>
<thead>
<tr class="contain-align-left">
Status
</th><th>
Instance
- </th><th colspan="2">
+ </th><th>
Template
</th><th>
Owner
<td>
<%= check_box_tag 'uuids[]', ob.uuid, false, :class => 'persistent-selection' %>
</td><td>
- <% if ob.success %>
- <span class="label label-success">success</span>
- <% elsif ob.active %>
- <span class="label label-info">active</span>
- <% end %>
- </td><td colspan="2">
+ <%= render partial: 'pipeline_status_label', locals: {:p => ob} %>
+ </td><td colspan="1">
<%= link_to_if_arvados_object ob, friendly_name: true %>
</td><td>
<%= link_to_if_arvados_object ob.pipeline_template_uuid, friendly_name: true %>
</td>
<td style="border-top: 0; opacity: 0.5;" colspan="5">
<% ob.components.each do |cname, c| %>
- <% status = if !(c.is_a?(Hash) && c[:job].is_a?(Hash))
- nil
- elsif c[:job][:success] == true
- 'success'
- elsif c[:job][:success] == false
- 'danger'
- elsif c[:job][:running] == true
- 'info'
- else
- 'warning'
- end %>
- <span class="label label-<%= status || 'default' %>"><%= cname.to_s %></span>
+ <% if c[:job] %>
+ <%= render partial: "job_status_label", locals: {:j => c[:job], :title => cname.to_s } %>
+ <% else %>
+ <span class="label label-default"><%= cname.to_s %></span>
+ <% end %>
<% end %>
</td>
</tr>
<% end %>
+<%= render partial: "paging", locals: {results: @objects, object: @object} %>
+
<% content_for :footer_js do %>
var showhide_compare = function() {
var form = $('form#compare')[0];
<% self.formats = [:html] %>
var new_content = "<%= escape_javascript(render template: 'pipeline_instances/show') %>";
-if ($('div.body-content').html() != new_content)
- $('div.body-content').html(new_content);
+var selected_tab_hrefs = [];
+if ($('div.body-content').html() != new_content) {
+ $('.nav-tabs li.active a').each(function() {
+ selected_tab_hrefs.push($(this).attr('href'));
+ });
+
+ $('div.body-content').html(new_content);
+
+ // Show the same tabs that were active before we rewrote body-content
+ $.each(selected_tab_hrefs, function(i, href) {
+ $('.nav-tabs li a[href="' + href + '"]').tab('show');
+ });
+}
+$(document).trigger('ajax:complete');
+<%= content_for :content_top do %>
+ <h2>Template '<%= @object.name %>'</h2>
+<% end %>
+
<table class="table topalign">
<thead>
</thead>
<tbody>
<% @object.attributes_for_display.each do |attr, attrvalue| %>
- <% if attr == 'components' and attrvalue.is_a? Hash and attrvalue[:steps].is_a? Array %>
-
- <tr><td>components[steps]</td><td>
- <table class="table">
- <% attrvalue[:steps].each_with_index do |s, i| %>
- <tr>
- <td><%= i %></td>
- <% %w(name function min_revision max_steps_per_node params).each do |key| %>
- <td>
- <% if key == 'params' %>
- <% s[key.to_sym].each do |p| %>
- <%= p[:name] %>
- <% if p[:data_locator] || p[:value] %>
- ← <%= p[:data_locator] || p[:value] %>
- <% end %>
- <%= '(optional)' if p[:optional] %>
- <br />
- <% end %>
- <% else %>
- <%= s[key.to_sym] %>
- <% end %>
- </td>
- <% end %>
- </tr>
- <% end %>
- </table>
- </td></tr>
-
- <% else %>
- <%= render partial: 'application/arvados_object_attr', locals: { attr: attr, attrvalue: attrvalue } %>
- <% end %>
+ <% if attr != 'components' %>
+ <%= render partial: 'application/arvados_object_attr', locals: { attr: attr, attrvalue: attrvalue } %>
+ <% end %>
<% end %>
</tbody>
</table>
--- /dev/null
+<% content_for :tab_line_buttons do %>
+ <%= form_tag '/pipeline_instances' do |f| %>
+ <%= hidden_field :pipeline_instance, :pipeline_template_uuid, :value => @object.uuid %>
+ <%= button_tag "Create pipeline", {class: 'btn btn-primary pull-right', id: "run-pipeline-button"} %>
+<% end %>
+<% end %>
+
+<%= render partial: 'pipeline_templates/show_components_template', locals: {:template => @object, :obj => nil} %>
--- /dev/null
+<table class="table pipeline-components-table" style="margin-top: -.1em">
+ <colgroup>
+ <col style="width: 15%" />
+ <col style="width: 20%" />
+ <col style="width: 20%" />
+ <col style="width: 45%" />
+ </colgroup>
+
+ <thead>
+ <tr>
+ <th>
+ component
+ </th><th>
+ script
+ </th><th>
+ parameter
+ </th><th>
+ value
+ </th>
+ </tr>
+ </thead>
+ <tbody>
+ <% order = PipelineTemplatesHelper::sort_components(template.components) %>
+ <% puts "order is #{order}" %>
+ <% order.each do |k| %>
+ <% template_value = template.components[k] %>
+ <% puts "#{k} #{template_value}" %>
+ <% if not template_value then next end %>
+ <tr>
+ <td><span class="label label-default"><%= k %></span></td>
+
+ <td><%= render_editable_subattribute obj, :components, [k, :script], template_value[:script] %></td>
+
+ <td>script version</td>
+
+ <td>
+ <%= render_editable_subattribute obj, :components, [k, :script_version], template_value[:script_version] %>
+ </td>
+ </tr>
+
+ <% if template_value[:script_parameters].length > 0 %>
+ <% template_value[:script_parameters].each do |p, tv| %>
+ <tr>
+ <td style="border-top: none"></td>
+ <td style="border-top: none"></td>
+
+ <td class="property-edit-row"><%= p %></td>
+ <td class="property-edit-row"><%= render_editable_subattribute obj, :components, [k, :script_parameters, p.to_sym], tv %></td>
+ <% end %>
+ </tr>
+ <% end %>
+ <% end %>
+ </tbody>
+</table>
--- /dev/null
+
+ <%= render partial: 'pipeline_instances/show_recent' %>
<% content_for :css do %>
.playbutton {
color: white;
- background: gray;
+ background: rgb(91, 192, 222);
border: 0px;
border-radius: 3px;
- padding: 0px 3px;
+ padding: 0px 3px;
}
.playbutton:hover {
- color: white;
- background: blackh;
+ background: rgb(57, 179, 215);
}
<% end %>
+<%= render partial: "paging", locals: {results: @objects, object: @object} %>
+
<table class="table table-hover">
<thead>
<tr class="contain-align-left">
<td>
<%= form_tag '/pipeline_instances' do |f| %>
<%= hidden_field :pipeline_instance, :pipeline_template_uuid, :value => ob.uuid %>
- <%= button_tag nil, {class: 'playbutton'} do %>
+ <%= button_tag nil, {class: 'playbutton', title: "Run #{ob.name}"} do %>
<span class="glyphicon glyphicon-play"></span>
<% end %>
<% end %>
</tbody>
</table>
+
+<%= render partial: "paging", locals: {results: @objects, object: @object} %>
--- /dev/null
+<div class="modal-dialog">
+ <div class="modal-content">
+
+ <%= form_tag setup_user_path, {id: 'setup_form', name: 'setup_form', method: 'get',
+ class: 'form-search', remote: true} do %>
+
+ <div class="modal-header">
+ <button type="button" class="close" onClick="reset_form()" data-dismiss="modal" aria-hidden="true">×</button>
+ <h4 class="modal-title">Setup User</h4>
+ </div>
+
+ <div class="modal-body">
+ <% if @object%>
+ <% uuid = @object.uuid %>
+ <% email = @object.email %>
+ <% end %>
+ <% disable_email = uuid != nil %>
+ <% identity_url_prefix = @current_selections[:identity_url_prefix] %>
+ <% disable_url_prefix = identity_url_prefix != nil %>
+ <% selected_repo = @current_selections[:repo_name] %>
+ <% selected_vm = @current_selections[:vm_uuid] %>
+
+ <input id="user_uuid" maxlength="250" name="user_uuid" type="hidden" value="<%=uuid%>">
+ <div class="form-group">
+ <label for="email">Email</label>
+ <% if disable_email %>
+ <input class="form-control" id="email" maxlength="250" name="email" type="text" value="<%=email%>" disabled>
+ <% else %>
+ <input class="form-control" id="email" maxlength="250" name="email" type="text">
+ <% end %>
+ </div>
+ <div class="form-group">
+ <label for="openid_prefix">Identity URL Prefix</label>
+ <% if disable_url_prefix %>
+ <input class="form-control" id="openid_prefix" maxlength="250" name="openid_prefix" type="text"
+ value="<%=identity_url_prefix%>" disabled=true>
+ <% else %>
+ <input class="form-control" id="openid_prefix" maxlength="250" name="openid_prefix" type="text"
+ value="<%= Rails.configuration.default_openid_prefix %>">
+ <% end %>
+ </div>
+ <div class="form-group">
+ <label for="repo_name">Repository Name</label>
+ <input class="form-control" id="repo_name" maxlength="250" name="repo_name" type="text" value="<%=selected_repo%>">
+ </div>
+ <div class="form-group">
+ <label for="vm_uuid">Virtual Machine</label>
+ <select class="form-control" name="vm_uuid">
+ <option value="" <%= 'selected' unless selected_vm %>>
+ Choose One:
+ </option>
+ <% @vms.each do |vm| %>
+ <option value="<%=vm.uuid%>"
+ <%= 'selected' if selected_vm == vm.uuid %>>
+ <%= vm.hostname %>
+ </option>
+ <% end %>
+ </select>
+ </div>
+ </div>
+
+ <div class="modal-footer">
+ <button type="submit" id="register" class="btn btn-primary" autofocus>Submit</button>
+ <button class="btn btn-default" onClick="reset_form()" data-dismiss="modal" aria-hidden="true">Cancel</button>
+ </div>
+
+ <% end #form %>
+ </div>
+</div>
--- /dev/null
+<p>
+ As an admin user, you can <%= link_to "view recent user activity", activity_users_url %>.
+</p>
+
--- /dev/null
+<p>As an admin, you can log in as this user. When you’ve
+finished, you will need to log out and log in again with your own
+account.</p>
+
+<blockquote>
+<%= button_to "Log in as #{@object.full_name}", sudo_user_url(id: @object.uuid), class: 'btn btn-primary' %>
+</blockquote>
+
+<p>As an admin, you can setup this user. Please input a VM and repository for the user. If you had previously provided any of these items, they are pre-filled for you and you can leave them as is if you would like to reuse them.</p>
+
+<blockquote>
+<%= link_to "Setup #{@object.full_name}", setup_popup_user_url(id: @object.uuid), {class: 'btn btn-primary', :remote => true, 'data-toggle' => "modal", 'data-target' => '#user-setup-modal-window'} %>
+</blockquote>
+
+<p>As an admin, you can deactivate and reset this user. This will remove all repository/VM permissions for the user. If you "setup" the user again, the user will have to sign the user agreement again.</p>
+
+<blockquote>
+<%= button_to "Deactivate #{@object.full_name}", unsetup_user_url(id: @object.uuid), class: 'btn btn-primary', confirm: "Are you sure you want to deactivate #{@object.full_name}?"%>
+</blockquote>
+
+<% content_for :footer_html do %>
+<div id="user-setup-modal-window" class="modal fade" role="dialog" aria-labelledby="myModalLabel" aria-hidden="true"></div>
+<% end %>
<col width="20%" />
<col width="20%" />
<col width="13%" />
- <col width="27%" />
+ <col width="13%" />
+ <col width="20%" />
</colgroup>
<tr>
<th>Log</th>
<th>Age</th>
<th>Status</th>
+ <th>Progress</th>
</tr>
<% @my_jobs[0..6].each do |j| %>
<td>
<small>
- <% if j.success %>
+ <% if j.success and j.output %>
<a href="<%= collection_path(j.output) %>">
<% Collection.limit(1).where(uuid: j.output).each do |c| %>
</td>
<td>
- <% if j.success %>
- <span class="label label-success">finished</span>
- <% elsif j.success == false %>
- <span class="label label-danger">failed</span>
- <% elsif j.finished_at %>
- <span class="label">finished?</span>
- <% elsif j.started_at %>
- <span class="label label-info">running</span>
- <% else %>
- <span class="label">queued</span>
- <% end %>
- <% percent_total_tasks = 100 / (j.tasks_summary[:running] + j.tasks_summary[:done] + j.tasks_summary[:todo]) rescue 0 %>
- <div class="inline-progress-container pull-right">
- <div class="progress">
- <span class="progress-bar progress-bar-success" style="width: <%= j.tasks_summary[:done] * percent_total_tasks rescue 0 %>%;">
- </span>
- <span class="progress-bar" style="width: <%= j.tasks_summary[:running] * percent_total_tasks rescue 0 %>%;">
- </span>
- <% if j.success == false %>
- <span class="progress-bar progress-bar-danger" style="width: <%= tasks_summary[:failed] * percent_total_tasks rescue 0 %>%;">
- </span>
- <% end %>
- </div>
+ <%= render partial: 'job_status_label', locals: {:j => j} %>
+</td>
+<td>
+ <div class="inline-progress-container">
+ <%= render partial: 'job_progress', locals: {:j => j} %>
</div>
</td>
<col width="30%" />
<col width="30%" />
<col width="13%" />
- <col width="27%" />
+ <col width="13%" />
+ <col width="20%" />
</colgroup>
<tr>
<th>Template</th>
<th>Age</th>
<th>Status</th>
+ <th>Progress</th>
</tr>
<% @my_pipelines[0..6].each do |p| %>
</td>
<td>
- <% if p.success %>
- <span class="label label-success">finished</span>
- <% elsif p.success == false %>
- <span class="label label-danger">failed</span>
- <% elsif p.active and p.modified_at < 30.minutes.ago %>
- <span class="label label-info">stopped</span>
- <% elsif p.active %>
- <span class="label label-info">running</span>
- <% else %>
- <span class="label">queued</span>
- <% end %>
+ <%= render partial: 'pipeline_status_label', locals: {:p => p} %>
+ </td>
- <% summary = pipeline_summary p %>
- <div class="inline-progress-container pull-right">
- <div class="progress">
- <span class="progress-bar progress-bar-success" style="width: <%= summary[:percent_done] %>%;">
- </span>
- <% if p.success == false %>
- <span class="progress-bar progress-bar-danger" style="width: <%= 100.0 - summary[:percent_done] %>%;">
- </span>
- <% else %>
- <span class="progress-bar" style="width: <%= summary[:percent_running] %>%;">
- </span>
- <span class="progress-bar progress-bar-info" style="width: <%= summary[:percent_queued] %>%;">
- </span>
- <span class="progress-bar progress-bar-danger" style="width: <%= summary[:percent_failed] %>%;">
- </span>
- <% end %>
- </div>
+ <td>
+ <div class="inline-progress-container">
+ <%= render partial: 'pipeline_progress', locals: {:p => p} %>
</div>
</td>
-
</tr>
<% end %>
</table>
<span class="glyphicon glyphicon-search"></span>
<% end %>
</span>
- </div>
+ </div>
<% end %>
</div>
<% if not current_user.andand.is_active or @my_collections.empty? %>
<p>
Your account must be activated by an Arvados administrator. If this
is your first time accessing Arvados and would like to request
- access, or you believe you are seeing the page in error, please
- <%= link_to "contact us", Rails.configuration.activation_contact_link %>.
+ access, or you believe you are seeing the page in error, please
+ <%= link_to "contact us", Rails.configuration.activation_contact_link %>.
You should receive an email at the address you used to log in when
- your account is activated. In the mean time, you can
+ your account is activated. In the mean time, you can
<%= link_to "learn more about Arvados", "https://arvados.org/projects/arvados/wiki/Introduction_to_Arvados" %>,
and <%= link_to "read the Arvados user guide", "http://doc.arvados.org/user" %>.
</p>
--- /dev/null
+<% content_for :css do %>
+table#users-activity-table th {
+ overflow-x: hidden;
+}
+table#users-activity-table .cell-for-span-This-month,
+table#users-activity-table .cell-for-span-Last-month {
+ background: #eee;
+}
+<% end %>
+<table class="table table-condensed arv-index" id="users-activity-table">
+ <colgroup>
+ <col width="28%" />
+ </colgroup>
+ <% @spans.each do |_| %>
+ <colgroup>
+ <% 3.times do %>
+ <col width="<%= (72 / @spans.count / 3).floor %>%" />
+ <% end %>
+ </colgroup>
+ <% end %>
+
+ <tr>
+ <th rowspan="2">User</th>
+ <% @spans.each do |span, start_at, end_at| %>
+ <th colspan="3" class="cell-for-span-<%= span.gsub ' ','-' %>">
+ <%= span %>
+ <br />
+ <%= start_at.strftime('%b %-d') %>
+ -
+ <%= (end_at-1.second).strftime('%b %-d') %>
+ </th>
+ <% end %>
+ </tr>
+ <tr>
+ <% @spans.each do |span, _| %>
+ <th class="cell-for-span-<%= span.gsub ' ','-' %>">Logins</th>
+ <th class="cell-for-span-<%= span.gsub ' ','-' %>">Jobs</th>
+ <th class="cell-for-span-<%= span.gsub ' ','-' %>">Pipelines</th>
+ <% end %>
+ </tr>
+
+ <% @users.each do |user| %>
+ <tr>
+ <td>
+ <small>
+ <% if user.uuid %>
+ <%= link_to_if_arvados_object user, friendly_name: true %>
+ <% else %>
+ <b>Total</b>
+ <% end %>
+ </small>
+ </td>
+
+ <% @spans.each do |span, _| %>
+ <% ['logins', 'jobs', 'pipeline_instances'].each do |type| %>
+ <td class="cell-for-span-<%= span.gsub ' ','-' %>">
+ <small>
+ <%= @user_activity[user.uuid][span + " " + type].to_s %>
+ </small>
+ </td>
+ <% end %>
+ <% end %>
+ </tr>
+ <% end %>
+</table>
+
+<% content_for :footer_js do %>
+$('#users-activity-table td small').each(function(){
+ if ($(this).html().trim() == '0')
+ $(this).css('opacity', '0.3');
+});
+<% end %>
--- /dev/null
+$("#user-setup-modal-window").modal("hide");
+document.location.reload();
--- /dev/null
+$("#user-setup-modal-window").html("<%= escape_javascript(render partial: 'setup_popup') %>");
+
+// disable the submit button on load
+var $input = $('input:text'),
+$register = $('#register');
+
+var email_disabled = document.forms["setup_form"]["email"].disabled;
+var email_value = document.forms["setup_form"]["email"].value;
+var prefix_value = document.forms["setup_form"]["openid_prefix"].value;
+if ((email_disabled == false) && (email_value == null || email_value == "" ||
+ prefix_value == null || prefix_value == "")) {
+ $register.attr('disabled', true);
+}
+
+// capture events to enable submit button when applicable
+$input.on('keyup paste mouseleave', function() {
+ var trigger = false;
+
+ var email_disabled = document.forms["setup_form"]["email"].disabled;
+ var email_value = document.forms["setup_form"]["email"].value;
+ var prefix_value = document.forms["setup_form"]["openid_prefix"].value;
+
+ var emailRegExp = /^([\w-\.]+@([\w-]+\.)+[\w-]{2,4})?$/;
+ var validEmail = false;
+
+ if (emailRegExp.test(email_value )) {
+ validEmail = true;
+ }
+
+ if ((email_disabled == false) && (!validEmail || email_value == null ||
+ email_value == "" || prefix_value == null || prefix_value == "")){
+ trigger = true;
+ }
+
+ trigger ? $register.attr('disabled', true) : $register.removeAttr('disabled');
+});
+
+// reset form input fields, for the next time around
+function reset_form() {
+ $('#email').val("");
+ $('#openid_prefix').val("");
+ $('#repo_name').val("");
+ $('select').val('')
+}
--- /dev/null
+# Do not use this file for site configuration. Create application.yml
+# instead (see application.yml.example).
+
+development:
+ cache_classes: false
+ whiny_nils: true
+ consider_all_requests_local: true
+ action_controller.perform_caching: false
+ action_mailer.raise_delivery_errors: false
+ active_support.deprecation: :log
+ action_dispatch.best_standards_support: :builtin
+ active_record.mass_assignment_sanitizer: :strict
+ active_record.auto_explain_threshold_in_seconds: 0.5
+ assets.compress: false
+ assets.debug: true
+ profiling_enabled: true
+ site_name: Arvados Workbench (dev)
+
+production:
+ force_ssl: true
+ cache_classes: true
+ consider_all_requests_local: false
+ action_controller.perform_caching: true
+ serve_static_assets: false
+ assets.compress: true
+ assets.compile: false
+ assets.digest: true
+ i18n.fallbacks: true
+ active_support.deprecation: :notify
+ profiling_enabled: false
+
+ arvados_insecure_https: false
+
+ data_import_dir: /data/arvados-workbench-upload/data
+ data_export_dir: /data/arvados-workbench-download/data
+
+ site_name: Arvados Workbench
+
+test:
+ cache_classes: true
+ serve_static_assets: true
+ static_cache_control: public, max-age=3600
+ whiny_nils: true
+ consider_all_requests_local: true
+ action_controller.perform_caching: false
+ action_dispatch.show_exceptions: false
+ action_controller.allow_forgery_protection: false
+ action_mailer.delivery_method: :test
+ active_record.mass_assignment_sanitizer: :strict
+ active_support.deprecation: :stderr
+ profiling_enabled: false
+ secret_token: <%= rand(2**256).to_s(36) %>
+
+ # When you run the Workbench's integration tests, it starts the API
+ # server as a dependency. These settings should match the API
+ # server's Rails defaults. If you adjust those, change these
+ # settings in application.yml to match.
+ arvados_login_base: https://localhost:3001/login
+ arvados_v1_base: https://localhost:3001/arvados/v1
+ arvados_insecure_https: true
+
+ site_name: Workbench:test
+
+common:
+ data_import_dir: /tmp/arvados-workbench-upload
+ data_export_dir: /tmp/arvados-workbench-download
+ arvados_login_base: https://arvados.local/login
+ arvados_v1_base: https://arvados.local/arvados/v1
+ arvados_insecure_https: true
+ activation_contact_link: mailto:info@arvados.org
+ arvados_docsite: http://doc.arvados.org
+ arvados_theme: default
+ show_user_agreement_inline: false
+ secret_token: ~
+ default_openid_prefix: https://www.google.com/accounts/o8/id
+ send_user_setup_notification_email: true
--- /dev/null
+# Copy this file to application.yml and edit to suit.
+#
+# Consult application.default.yml for the full list of configuration
+# settings.
+#
+# The order of precedence is:
+# 1. config/environments/{RAILS_ENV}.rb (deprecated)
+# 2. Section in application.yml corresponding to RAILS_ENV (e.g., development)
+# 3. Section in application.yml called "common"
+# 4. Section in application.default.yml corresponding to RAILS_ENV
+# 5. Section in application.default.yml called "common"
+
+common:
+ # At minimum, you need a nice long randomly generated secret_token here.
+ secret_token: ~
+
+ # You probably also want to point to your API server.
+ arvados_login_base: https://arvados.local:3000/login
+ arvados_v1_base: https://arvados.local:3000/arvados/v1
+ arvados_insecure_https: true
# Expands the lines which load the assets
config.assets.debug = true
- # Log timing data for API transactions
- config.profiling_enabled = true
-
- config.arvados_login_base = 'http://arvados.local/login'
- config.arvados_v1_base = 'http://arvados.local/arvados/v1'
- config.arvados_insecure_https = true # true = do not check server certificate
-
- config.data_import_dir = '/tmp/arvados-workbench-upload'
- config.data_export_dir = '/tmp/arvados-workbench-download'
-
- config.secret_token = File.read('config/.secret_token') if File.exist? 'config/.secret_token'
-
- config.site_name = 'Arvados Workbench (dev)'
- config.activation_contact_link = 'mailto:info@arvados.org'
-
- config.arvados_docsite = 'http://doc.arvados.org'
-
- config.arvados_theme = 'default'
-
- config.show_user_agreement_inline = false
end
# Log timing data for API transactions
config.profiling_enabled = false
- config.arvados_login_base = 'https://arvados.local/login'
- config.arvados_v1_base = 'https://arvados.local/arvados/v1'
- config.arvados_insecure_https = false # true = do not check server certificate
-
- config.data_import_dir = '/data/arvados-workbench-upload/data'
- config.data_export_dir = '/data/arvados-workbench-download/data'
-
- # Authentication stub: hard code pre-approved API tokens.
- # config.accept_api_token = { rand(2**256).to_s(36) => true }
- config.accept_api_token = {}
-
- config.site_name = 'Arvados Workbench'
- config.activation_contact_link = 'mailto:info@arvados.org'
-
- config.arvados_docsite = 'http://doc.arvados.org'
-
- config.arvados_theme = 'default'
-
- config.show_user_agreement_inline = false
end
# Log timing data for API transactions
config.profiling_enabled = false
- config.arvados_login_base = 'http://arvados.local/login'
- config.arvados_v1_base = 'https://arvados.local/arvados/v1'
- config.arvados_insecure_https = true # true = do not check server certificate
-
- config.data_import_dir = '/data/arvados-workbench-upload'
- config.data_export_dir = '/data/arvados-workbench-download'
-
- # Authentication stub: hard code pre-approved API tokens.
- # config.accept_api_token = { rand(2**256).to_s(36) => true }
- config.accept_api_token = {}
-
- config.site_name = 'Arvados Workbench (test)'
- config.activation_contact_link = 'mailto:info@arvados.org'
-
- config.arvados_docsite = 'http://doc.arvados.org'
-
- config.arvados_theme = 'default'
-
- config.show_user_agreement_inline = false
end
+++ /dev/null
-$arvados_api_client = ArvadosApiClient.new
--- /dev/null
+# This file must be loaded _after_ secret_token.rb if secret_token is
+# defined there instead of in config/application.yml.
+
+$application_config = {}
+
+%w(application.default application).each do |cfgfile|
+ path = "#{::Rails.root.to_s}/config/#{cfgfile}.yml"
+ if File.exists? path
+ yaml = ERB.new(IO.read path).result(binding)
+ confs = YAML.load(yaml)
+ $application_config.merge!(confs['common'] || {})
+ $application_config.merge!(confs[::Rails.env.to_s] || {})
+ end
+end
+
+ArvadosWorkbench::Application.configure do
+ nils = []
+ $application_config.each do |k, v|
+ # "foo.bar: baz" --> { config.foo.bar = baz }
+ cfg = config
+ ks = k.split '.'
+ k = ks.pop
+ ks.each do |kk|
+ cfg = cfg.send(kk)
+ end
+ if cfg.respond_to?(k.to_sym) and !cfg.send(k).nil?
+ # Config must have been set already in environments/*.rb.
+ #
+ # After config files have been migrated, this mechanism should
+ # be deprecated, then removed.
+ elsif v.nil?
+ # Config variables are not allowed to be nil. Make a "naughty"
+ # list, and present it below.
+ nils << k
+ else
+ cfg.send "#{k}=", v
+ end
+ end
+ if !nils.empty?
+ raise <<EOS
+Refusing to start in #{::Rails.env.to_s} mode with missing configuration.
+
+The following configuration settings must be specified in
+config/application.yml:
+* #{nils.join "\n* "}
+
+EOS
+ end
+end
--- /dev/null
+# The client object must be instantiated _after_ zza_load_config.rb
+# runs, because it relies on configuration settings.
+#
+if not $application_config
+ raise "Fatal: Config must be loaded before instantiating ArvadosApiClient."
+end
+
+$arvados_api_client = ArvadosApiClient.new
themes_for_rails
resources :keep_disks
- resources :user_agreements
- post '/user_agreements/sign' => 'user_agreements#sign'
+ resources :user_agreements do
+ put 'sign', on: :collection
+ get 'signatures', on: :collection
+ end
get '/user_agreements/signatures' => 'user_agreements#signatures'
+ get "users/setup_popup" => 'users#setup_popup', :as => :setup_user_popup
+ get "users/setup" => 'users#setup', :as => :setup_user
resources :nodes
resources :humans
resources :traits
resources :users do
get 'home', :on => :member
get 'welcome', :on => :collection
+ get 'activity', :on => :collection
+ post 'sudo', :on => :member
+ post 'unsetup', :on => :member
+ get 'setup_popup', :on => :member
end
resources :logs
resources :factory_jobs
match '/collections/graph' => 'collections#graph'
resources :collections
get '/collections/:uuid/*file' => 'collections#show_file', :format => false
+
+ post 'actions' => 'actions#post'
+
root :to => 'users#welcome'
# Send unroutable requests to an arbitrary controller
+++ /dev/null
-Use this README file to introduce your application and point to useful places in the API for learning more.
-Run "rake doc:app" to generate API documentation for your models, controllers, helpers, and libraries.
--- /dev/null
+namespace :config do
+ desc 'Ensure site configuration has all required settings'
+ task check: :environment do
+ $application_config.sort.each do |k, v|
+ if ENV.has_key?('QUIET') then
+ # Make sure we still check for the variable to exist
+ eval("Rails.configuration.#{k}")
+ else
+ if /(password|secret)/.match(k) then
+ # Make sure we still check for the variable to exist, but don't print the value
+ eval("Rails.configuration.#{k}")
+ $stderr.puts "%-32s %s" % [k, '*********']
+ else
+ $stderr.puts "%-32s %s" % [k, eval("Rails.configuration.#{k}")]
+ end
+ end
+ end
+ end
+end
require 'test_helper'
class UsersControllerTest < ActionController::TestCase
+ test "ignore previously valid token (for deleted user), don't crash" do
+ get :welcome, {}, session_for(:valid_token_deleted_user)
+ assert_response :success
+ assert_nil assigns(:my_jobs)
+ assert_nil assigns(:my_ssh_keys)
+ end
+
+ test "expired token redirects to api server login" do
+ get :show, {
+ id: api_fixture('users')['active']['uuid']
+ }, session_for(:expired_trustedclient)
+ assert_response :redirect
+ assert_match /^#{Rails.configuration.arvados_login_base}/, @response.redirect_url
+ assert_nil assigns(:my_jobs)
+ assert_nil assigns(:my_ssh_keys)
+ end
end
--- /dev/null
+require 'test_helper'
+
+class LoginsTest < ActionDispatch::IntegrationTest
+ test "login with api_token works after redirect" do
+ visit page_with_token('active_trustedclient')
+ assert page.has_text?('Recent jobs'), "Missing 'Recent jobs' from page"
+ assert_no_match(/\bapi_token=/, current_path)
+ end
+
+ test "can't use expired token" do
+ visit page_with_token('expired_trustedclient')
+ assert page.has_text? 'Log in'
+ end
+
+ test "expired token yields login page, not error page" do
+ skip
+ visit page_with_token('expired_trustedclient')
+ # Even the error page has a "Log in" link. We should look for
+ # something that only appears the real login page.
+ assert page.has_text? 'Please log in'
+ end
+end
--- /dev/null
+require 'integration_helper'
+require 'uri'
+
+class SmokeTest < ActionDispatch::IntegrationTest
+ def assert_visit_success(allowed=[200])
+ assert_includes(allowed, status_code,
+ "#{current_url} returned #{status_code}, not one of " +
+ allowed.inspect)
+ end
+
+ def all_links_in(find_spec, text_regexp=//)
+ find(find_spec).all('a').collect { |tag|
+ if tag[:href].nil? or tag[:href].empty? or (tag.text !~ text_regexp)
+ nil
+ else
+ url = URI(tag[:href])
+ url.host.nil? ? url.path : nil
+ end
+ }.compact
+ end
+
+ test "all first-level links succeed" do
+ visit page_with_token('active_trustedclient', '/')
+ assert_visit_success
+ click_link 'user-menu'
+ urls = [all_links_in('.arvados-nav'),
+ all_links_in('.navbar', /^Manage /)].flatten
+ seen_urls = ['/']
+ while not (url = urls.shift).nil?
+ next if seen_urls.include? url
+ visit url
+ seen_urls << url
+ assert_visit_success
+ # Uncommenting the line below lets you crawl the entire site for a
+ # more thorough test.
+ # urls += all_links_in('body')
+ end
+ end
+end
--- /dev/null
+require 'integration_helper'
+require 'selenium-webdriver'
+require 'headless'
+
+class UsersTest < ActionDispatch::IntegrationTest
+
+ test "login as active user but not admin" do
+ Capybara.current_driver = Capybara.javascript_driver
+ visit page_with_token('active_trustedclient')
+
+ assert page.has_no_link? 'Users' 'Found Users link for non-admin user'
+ end
+
+ test "login as admin user and verify active user data" do
+ Capybara.current_driver = Capybara.javascript_driver
+ visit page_with_token('admin_trustedclient')
+
+ # go to Users list page
+ click_link 'Users'
+
+ # check active user attributes in the list page
+ page.within(:xpath, '//tr[@data-object-uuid="zzzzz-tpzed-xurymjxw79nv3jz"]') do
+ assert (text.include? 'true false'), 'Expected is_active'
+ end
+
+ click_link 'zzzzz-tpzed-xurymjxw79nv3jz'
+ assert page.has_text? 'Attributes'
+ assert page.has_text? 'Metadata'
+ assert page.has_text? 'Admin'
+
+ # go to the Attributes tab
+ click_link 'Attributes'
+ assert page.has_text? 'modified_by_user_uuid'
+ page.within(:xpath, '//a[@data-name="is_active"]') do
+ assert_equal "true", text, "Expected user's is_active to be true"
+ end
+ page.within(:xpath, '//a[@data-name="is_admin"]') do
+ assert_equal "false", text, "Expected user's is_admin to be false"
+ end
+
+ end
+
+ test "create a new user" do
+ headless = Headless.new
+ headless.start
+
+ Capybara.current_driver = :selenium
+
+ visit page_with_token('admin_trustedclient')
+
+ click_link 'Users'
+
+ assert page.has_text? 'zzzzz-tpzed-d9tiejq69daie8f'
+
+ click_link 'Add a new user'
+
+ sleep(0.1)
+ popup = page.driver.browser.window_handles.last
+ page.within_window popup do
+ assert has_text? 'Virtual Machine'
+ fill_in "email", :with => "foo@example.com"
+ fill_in "repo_name", :with => "test_repo"
+ click_button "Submit"
+ end
+
+ sleep(0.1)
+
+ # verify that the new user showed up in the users page
+ assert page.has_text? 'foo@example.com'
+
+ new_user_uuid = nil
+ all("tr").each do |elem|
+ if elem.text.include? 'foo@example.com'
+ new_user_uuid = elem.text.split[0]
+ break
+ end
+ end
+
+ assert new_user_uuid, "Expected new user uuid not found"
+
+ # go to the new user's page
+ click_link new_user_uuid
+
+ assert page.has_text? 'modified_by_user_uuid'
+ page.within(:xpath, '//a[@data-name="is_active"]') do
+ assert_equal "false", text, "Expected new user's is_active to be false"
+ end
+
+ click_link 'Metadata'
+ assert page.has_text? '(Repository: test_repo)'
+ assert !(page.has_text? '(VirtualMachine:)')
+
+ headless.stop
+ end
+
+ test "setup the active user" do
+ headless = Headless.new
+ headless.start
+
+ Capybara.current_driver = :selenium
+ visit page_with_token('admin_trustedclient')
+
+ click_link 'Users'
+
+ assert page.has_link? 'zzzzz-tpzed-xurymjxw79nv3jz'
+
+ # click on active user
+ click_link 'zzzzz-tpzed-xurymjxw79nv3jz'
+
+ # Setup user
+ click_link 'Admin'
+ assert page.has_text? 'As an admin, you can setup'
+
+ click_link 'Setup Active User'
+
+ sleep(0.1)
+
+ popup = page.driver.browser.window_handles.last
+ page.within_window popup do
+ assert has_text? 'Virtual Machine'
+ fill_in "repo_name", :with => "test_repo"
+ click_button "Submit"
+ end
+
+ sleep(0.1)
+ assert page.has_text? 'modified_by_client_uuid'
+
+ click_link 'Metadata'
+ assert page.has_text? '(Repository: test_repo)'
+ assert !(page.has_text? '(VirtualMachine:)')
+
+ # Click on Setup button again and this time also choose a VM
+ click_link 'Admin'
+ click_link 'Setup Active User'
+
+ sleep(0.1)
+ popup = page.driver.browser.window_handles.last
+ page.within_window popup do
+ fill_in "repo_name", :with => "second_test_repo"
+ select("testvm.shell", :from => 'vm_uuid')
+ click_button "Submit"
+ end
+
+ sleep(0.1)
+ assert page.has_text? 'modified_by_client_uuid'
+
+ click_link 'Metadata'
+ assert page.has_text? '(Repository: second_test_repo)'
+ assert page.has_text? '(VirtualMachine: testvm.shell)'
+
+ headless.stop
+ end
+
+ test "unsetup active user" do
+ headless = Headless.new
+ headless.start
+
+ Capybara.current_driver = :selenium
+
+ visit page_with_token('admin_trustedclient')
+
+ click_link 'Users'
+
+ assert page.has_link? 'zzzzz-tpzed-xurymjxw79nv3jz'
+
+ # click on active user
+ click_link 'zzzzz-tpzed-xurymjxw79nv3jz'
+
+ # Verify that is_active is set
+ click_link 'Attributes'
+ assert page.has_text? 'modified_by_user_uuid'
+ page.within(:xpath, '//a[@data-name="is_active"]') do
+ assert_equal "true", text, "Expected user's is_active to be true"
+ end
+
+ # go to Admin tab
+ click_link 'Admin'
+ assert page.has_text? 'As an admin, you can deactivate and reset this user'
+
+ # unsetup user and verify all the above links are deleted
+ click_link 'Admin'
+ click_button 'Deactivate Active User'
+ sleep(0.1)
+
+ # Should now be back in the Attributes tab for the user
+ page.driver.browser.switch_to.alert.accept
+ assert page.has_text? 'modified_by_user_uuid'
+ page.within(:xpath, '//a[@data-name="is_active"]') do
+ assert_equal "false", text, "Expected user's is_active to be false after unsetup"
+ end
+
+ click_link 'Metadata'
+ assert !(page.has_text? '(Repository: test_repo)')
+ assert !(page.has_text? '(Repository: second_test_repo)')
+ assert !(page.has_text? '(VirtualMachine: testvm.shell)')
+
+ # setup user again and verify links present
+ click_link 'Admin'
+ click_link 'Setup Active User'
+
+ sleep(0.1)
+ popup = page.driver.browser.window_handles.last
+ page.within_window popup do
+ fill_in "repo_name", :with => "second_test_repo"
+ select("testvm.shell", :from => 'vm_uuid')
+ click_button "Submit"
+ end
+
+ sleep(0.1)
+ assert page.has_text? 'modified_by_client_uuid'
+
+ click_link 'Metadata'
+ assert page.has_text? '(Repository: second_test_repo)'
+ assert page.has_text? '(VirtualMachine: testvm.shell)'
+
+ headless.stop
+ end
+
+end
--- /dev/null
+require 'integration_helper'
+
+class VirtualMachinesTest < ActionDispatch::IntegrationTest
+ test "make and name a new virtual machine" do
+ Capybara.current_driver = Capybara.javascript_driver
+ visit page_with_token('admin_trustedclient')
+ click_link 'Virtual machines'
+ assert page.has_text? 'testvm.shell'
+ click_on 'Add a new virtual machine'
+ assert page.has_text? 'none'
+ click_link 'none'
+ assert page.has_text? 'Update hostname'
+ fill_in 'editable-text', with: 'testname'
+ click_button 'editable-submit'
+ assert page.has_text? 'testname'
+ end
+end
--- /dev/null
+require 'test_helper'
+require 'capybara/rails'
+require 'capybara/poltergeist'
+require 'uri'
+require 'yaml'
+
+class ActionDispatch::IntegrationTest
+ # Make the Capybara DSL available in all integration tests
+ include Capybara::DSL
+ include ApiFixtureLoader
+
+ @@API_AUTHS = self.api_fixture('api_client_authorizations')
+
+ def page_with_token(token, path='/')
+ # Generate a page path with an embedded API token.
+ # Typical usage: visit page_with_token('token_name', page)
+ # The token can be specified by the name of an api_client_authorizations
+ # fixture, or passed as a raw string.
+ api_token = ((@@API_AUTHS.include? token) ?
+ @@API_AUTHS[token]['api_token'] : token)
+ sep = (path.include? '?') ? '&' : '?'
+ q_string = URI.encode_www_form('api_token' => api_token)
+ "#{path}#{sep}#{q_string}"
+ end
+end
require File.expand_path('../../config/environment', __FILE__)
require 'rails/test_help'
+$ARV_API_SERVER_DIR = File.expand_path('../../../../services/api', __FILE__)
+SERVER_PID_PATH = 'tmp/pids/server.pid'
+
class ActiveSupport::TestCase
# Setup all fixtures in test/fixtures/*.(yml|csv) for all tests in alphabetical order.
#
# Add more helper methods to be used by all tests here...
end
+
+module ApiFixtureLoader
+ def self.included(base)
+ base.extend(ClassMethods)
+ end
+
+ module ClassMethods
+ @@api_fixtures = {}
+ def api_fixture(name)
+ # Returns the data structure from the named API server test fixture.
+ @@api_fixtures[name] ||= \
+ begin
+ path = File.join($ARV_API_SERVER_DIR, 'test', 'fixtures', "#{name}.yml")
+ YAML.load(IO.read(path))
+ end
+ end
+ end
+ def api_fixture name
+ self.class.api_fixture name
+ end
+end
+
+class ActiveSupport::TestCase
+ include ApiFixtureLoader
+ def session_for api_client_auth_name
+ {
+ arvados_api_token: api_fixture('api_client_authorizations')[api_client_auth_name.to_s]['api_token']
+ }
+ end
+end
+
+class ApiServerBackedTestRunner < MiniTest::Unit
+ # Make a hash that unsets Bundle's environment variables.
+ # We'll use this environment when we launch Bundle commands in the API
+ # server. Otherwise, those commands will try to use Workbench's gems, etc.
+ @@APIENV = Hash[ENV.map { |key, val|
+ (key =~ /^BUNDLE_/) ? [key, nil] : nil
+ }.compact]
+
+ def _system(*cmd)
+ if not system(@@APIENV, *cmd)
+ raise RuntimeError, "#{cmd[0]} returned exit code #{$?.exitstatus}"
+ end
+ end
+
+ def _run(args=[])
+ Capybara.javascript_driver = :poltergeist
+ server_pid = Dir.chdir($ARV_API_SERVER_DIR) do |apidir|
+ _system('bundle', 'exec', 'rake', 'db:test:load')
+ _system('bundle', 'exec', 'rake', 'db:fixtures:load')
+ _system('bundle', 'exec', 'rails', 'server', '-d')
+ timeout = Time.now.tv_sec + 10
+ begin
+ sleep 0.2
+ begin
+ server_pid = IO.read(SERVER_PID_PATH).to_i
+ good_pid = (server_pid > 0) and (Process.kill(0, pid) rescue false)
+ rescue Errno::ENOENT
+ good_pid = false
+ end
+ end while (not good_pid) and (Time.now.tv_sec < timeout)
+ if not good_pid
+ raise RuntimeError, "could not find API server Rails pid"
+ end
+ server_pid
+ end
+ begin
+ super(args)
+ ensure
+ Process.kill('TERM', server_pid)
+ end
+ end
+end
+
+MiniTest::Unit.runner = ApiServerBackedTestRunner.new
require 'test_helper'
class CollectionTest < ActiveSupport::TestCase
- # test "the truth" do
- # assert true
- # end
+ test 'recognize empty blob locator' do
+ ['d41d8cd98f00b204e9800998ecf8427e+0',
+ 'd41d8cd98f00b204e9800998ecf8427e',
+ 'd41d8cd98f00b204e9800998ecf8427e+0+Xyzzy'].each do |x|
+ assert_equal true, Collection.is_empty_blob_locator?(x)
+ end
+ ['d41d8cd98f00b204e9800998ecf8427e0',
+ 'acbd18db4cc2f85cedef654fccc4a4d8+3',
+ 'acbd18db4cc2f85cedef654fccc4a4d8+0'].each do |x|
+ assert_equal false, Collection.is_empty_blob_locator?(x)
+ end
+ end
end
--- /dev/null
+Creative Commons Legal Code
+
+Attribution-ShareAlike 3.0 United States
+
+License
+
+THE WORK (AS DEFINED BELOW) IS PROVIDED UNDER THE TERMS OF THIS CREATIVE
+COMMONS PUBLIC LICENSE ("CCPL" OR "LICENSE"). THE WORK IS PROTECTED BY
+COPYRIGHT AND/OR OTHER APPLICABLE LAW. ANY USE OF THE WORK OTHER THAN AS
+AUTHORIZED UNDER THIS LICENSE OR COPYRIGHT LAW IS PROHIBITED.
+
+BY EXERCISING ANY RIGHTS TO THE WORK PROVIDED HERE, YOU ACCEPT AND AGREE TO BE
+BOUND BY THE TERMS OF THIS LICENSE. TO THE EXTENT THIS LICENSE MAY BE
+CONSIDERED TO BE A CONTRACT, THE LICENSOR GRANTS YOU THE RIGHTS CONTAINED HERE
+IN CONSIDERATION OF YOUR ACCEPTANCE OF SUCH TERMS AND CONDITIONS.
+
+1. Definitions
+
+ a. "Collective Work" means a work, such as a periodical issue, anthology or
+ encyclopedia, in which the Work in its entirety in unmodified form, along
+ with one or more other contributions, constituting separate and independent
+ works in themselves, are assembled into a collective whole. A work that
+ constitutes a Collective Work will not be considered a Derivative Work (as
+ defined below) for the purposes of this License.
+
+ b. "Creative Commons Compatible License" means a license that is listed at
+ http://creativecommons.org/compatiblelicenses that has been approved by
+ Creative Commons as being essentially equivalent to this License,
+ including, at a minimum, because that license: (i) contains terms that have
+ the same purpose, meaning and effect as the License Elements of this
+ License; and, (ii) explicitly permits the relicensing of derivatives of
+ works made available under that license under this License or either a
+ Creative Commons unported license or a Creative Commons jurisdiction
+ license with the same License Elements as this License.
+
+ c. "Derivative Work" means a work based upon the Work or upon the Work and
+ other pre-existing works, such as a translation, musical arrangement,
+ dramatization, fictionalization, motion picture version, sound recording,
+ art reproduction, abridgment, condensation, or any other form in which the
+ Work may be recast, transformed, or adapted, except that a work that
+ constitutes a Collective Work will not be considered a Derivative Work for
+ the purpose of this License. For the avoidance of doubt, where the Work is
+ a musical composition or sound recording, the synchronization of the Work
+ in timed-relation with a moving image ("synching") will be considered a
+ Derivative Work for the purpose of this License.
+
+ d. "License Elements" means the following high-level license attributes as
+ selected by Licensor and indicated in the title of this License:
+ Attribution, ShareAlike.
+
+ e. "Licensor" means the individual, individuals, entity or entities that
+ offers the Work under the terms of this License.
+
+ f. "Original Author" means the individual, individuals, entity or entities who
+ created the Work.
+
+ g. "Work" means the copyrightable work of authorship offered under the terms
+ of this License.
+
+ h. "You" means an individual or entity exercising rights under this License
+ who has not previously violated the terms of this License with respect to
+ the Work, or who has received express permission from the Licensor to
+ exercise rights under this License despite a previous violation.
+
+2. Fair Use Rights. Nothing in this license is intended to reduce, limit, or
+restrict any rights arising from fair use, first sale or other limitations on
+the exclusive rights of the copyright owner under copyright law or other
+applicable laws.
+
+3. License Grant. Subject to the terms and conditions of this License, Licensor
+hereby grants You a worldwide, royalty-free, non-exclusive, perpetual (for the
+duration of the applicable copyright) license to exercise the rights in the
+Work as stated below:
+
+ a. to reproduce the Work, to incorporate the Work into one or more Collective
+ Works, and to reproduce the Work as incorporated in the Collective Works;
+
+ b. to create and reproduce Derivative Works provided that any such
+ Derivative Work, including any translation in any medium, takes reasonable
+ steps to clearly label, demarcate or otherwise identify that changes were
+ made to the original Work. For example, a translation could be marked "The
+ original work was translated from English to Spanish," or a modification
+ could indicate "The original work has been modified.";
+
+ c. to distribute copies or phonorecords of, display publicly, perform
+ publicly, and perform publicly by means of a digital audio transmission the
+ Work including as incorporated in Collective Works;
+
+ d. to distribute copies or phonorecords of, display publicly, perform
+ publicly, and perform publicly by means of a digital audio transmission
+ Derivative Works.
+
+ e. For the avoidance of doubt, where the Work is a musical composition:
+
+ i. Performance Royalties Under Blanket Licenses. Licensor waives the
+ exclusive right to collect, whether individually or, in the event that
+ Licensor is a member of a performance rights society (e.g. ASCAP, BMI,
+ SESAC), via that society, royalties for the public performance or
+ public digital performance (e.g. webcast) of the Work.
+
+ ii. Mechanical Rights and Statutory Royalties. Licensor waives the
+ exclusive right to collect, whether individually or via a music rights
+ agency or designated agent (e.g. Harry Fox Agency), royalties for any
+ phonorecord You create from the Work ("cover version") and distribute,
+ subject to the compulsory license created by 17 USC Section 115 of the
+ US Copyright Act (or the equivalent in other jurisdictions).
+
+ f. Webcasting Rights and Statutory Royalties. For the avoidance of doubt,
+ where the Work is a sound recording, Licensor waives the exclusive right to
+ collect, whether individually or via a performance-rights society
+ (e.g. SoundExchange), royalties for the public digital performance
+ (e.g. webcast) of the Work, subject to the compulsory license created by 17
+ USC Section 114 of the US Copyright Act (or the equivalent in other
+ jurisdictions).
+
+The above rights may be exercised in all media and formats whether now known or
+hereafter devised. The above rights include the right to make such
+modifications as are technically necessary to exercise the rights in other
+media and formats. All rights not expressly granted by Licensor are hereby
+reserved.
+
+4. Restrictions. The license granted in Section 3 above is expressly made subject to and limited by the following restrictions:
+
+ a. You may distribute, publicly display, publicly perform, or publicly
+ digitally perform the Work only under the terms of this License, and You
+ must include a copy of, or the Uniform Resource Identifier for, this
+ License with every copy or phonorecord of the Work You distribute, publicly
+ display, publicly perform, or publicly digitally perform. You may not offer
+ or impose any terms on the Work that restrict the terms of this License or
+ the ability of a recipient of the Work to exercise of the rights granted to
+ that recipient under the terms of the License. You may not sublicense the
+ Work. You must keep intact all notices that refer to this License and to
+ the disclaimer of warranties. When You distribute, publicly display,
+ publicly perform, or publicly digitally perform the Work, You may not
+ impose any technological measures on the Work that restrict the ability of
+ a recipient of the Work from You to exercise of the rights granted to that
+ recipient under the terms of the License. This Section 4(a) applies to the
+ Work as incorporated in a Collective Work, but this does not require the
+ Collective Work apart from the Work itself to be made subject to the terms
+ of this License. If You create a Collective Work, upon notice from any
+ Licensor You must, to the extent practicable, remove from the Collective
+ Work any credit as required by Section 4(c), as requested. If You create a
+ Derivative Work, upon notice from any Licensor You must, to the extent
+ practicable, remove from the Derivative Work any credit as required by
+ Section 4(c), as requested.
+
+ b. You may distribute, publicly display, publicly perform, or publicly
+ digitally perform a Derivative Work only under: (i) the terms of this
+ License; (ii) a later version of this License with the same License
+ Elements as this License; (iii) either the Creative Commons (Unported)
+ license or a Creative Commons jurisdiction license (either this or a later
+ license version) that contains the same License Elements as this License
+ (e.g. Attribution-ShareAlike 3.0 (Unported)); (iv) a Creative Commons
+ Compatible License. If you license the Derivative Work under one of the
+ licenses mentioned in (iv), you must comply with the terms of that
+ license. If you license the Derivative Work under the terms of any of the
+ licenses mentioned in (i), (ii) or (iii) (the "Applicable License"), you
+ must comply with the terms of the Applicable License generally and with the
+ following provisions: (I) You must include a copy of, or the Uniform
+ Resource Identifier for, the Applicable License with every copy or
+ phonorecord of each Derivative Work You distribute, publicly display,
+ publicly perform, or publicly digitally perform; (II) You may not offer or
+ impose any terms on the Derivative Works that restrict the terms of the
+ Applicable License or the ability of a recipient of the Work to exercise
+ the rights granted to that recipient under the terms of the Applicable
+ License; (III) You must keep intact all notices that refer to the
+ Applicable License and to the disclaimer of warranties; and, (IV) when You
+ distribute, publicly display, publicly perform, or publicly digitally
+ perform the Work, You may not impose any technological measures on the
+ Derivative Work that restrict the ability of a recipient of the Derivative
+ Work from You to exercise the rights granted to that recipient under the
+ terms of the Applicable License. This Section 4(b) applies to the
+ Derivative Work as incorporated in a Collective Work, but this does not
+ require the Collective Work apart from the Derivative Work itself to be
+ made subject to the terms of the Applicable License.
+
+ c. If You distribute, publicly display, publicly perform, or publicly
+ digitally perform the Work (as defined in Section 1 above) or any
+ Derivative Works (as defined in Section 1 above) or Collective Works (as
+ defined in Section 1 above), You must, unless a request has been made
+ pursuant to Section 4(a), keep intact all copyright notices for the Work
+ and provide, reasonable to the medium or means You are utilizing: (i) the
+ name of the Original Author (or pseudonym, if applicable) if supplied,
+ and/or (ii) if the Original Author and/or Licensor designate another party
+ or parties (e.g. a sponsor institute, publishing entity, journal) for
+ attribution ("Attribution Parties") in Licensor's copyright notice, terms
+ of service or by other reasonable means, the name of such party or parties;
+ the title of the Work if supplied; to the extent reasonably practicable,
+ the Uniform Resource Identifier, if any, that Licensor specifies to be
+ associated with the Work, unless such URI does not refer to the copyright
+ notice or licensing information for the Work; and, consistent with Section
+ 3(b) in the case of a Derivative Work, a credit identifying the use of the
+ Work in the Derivative Work (e.g., "French translation of the Work by
+ Original Author," or "Screenplay based on original Work by Original
+ Author"). The credit required by this Section 4(c) may be implemented in
+ any reasonable manner; provided, however, that in the case of a Derivative
+ Work or Collective Work, at a minimum such credit will appear, if a credit
+ for all contributing authors of the Derivative Work or Collective Work
+ appears, then as part of these credits and in a manner at least as
+ prominent as the credits for the other contributing authors. For the
+ avoidance of doubt, You may only use the credit required by this Section
+ for the purpose of attribution in the manner set out above and, by
+ exercising Your rights under this License, You may not implicitly or
+ explicitly assert or imply any connection with, sponsorship or endorsement
+ by the Original Author, Licensor and/or Attribution Parties, as
+ appropriate, of You or Your use of the Work, without the separate, express
+ prior written permission of the Original Author, Licensor and/or
+ Attribution Parties.
+
+
+5. Representations, Warranties and Disclaimer
+
+UNLESS OTHERWISE MUTUALLY AGREED TO BY THE PARTIES IN WRITING, LICENSOR OFFERS
+THE WORK AS-IS AND ONLY TO THE EXTENT OF ANY RIGHTS HELD IN THE LICENSED WORK
+BY THE LICENSOR. THE LICENSOR MAKES NO REPRESENTATIONS OR WARRANTIES OF ANY
+KIND CONCERNING THE WORK, EXPRESS, IMPLIED, STATUTORY OR OTHERWISE, INCLUDING,
+WITHOUT LIMITATION, WARRANTIES OF TITLE, MARKETABILITY, MERCHANTIBILITY,
+FITNESS FOR A PARTICULAR PURPOSE, NONINFRINGEMENT, OR THE ABSENCE OF LATENT OR
+OTHER DEFECTS, ACCURACY, OR THE PRESENCE OF ABSENCE OF ERRORS, WHETHER OR NOT
+DISCOVERABLE. SOME JURISDICTIONS DO NOT ALLOW THE EXCLUSION OF IMPLIED
+WARRANTIES, SO SUCH EXCLUSION MAY NOT APPLY TO YOU.
+
+6. Limitation on Liability. EXCEPT TO THE EXTENT REQUIRED BY APPLICABLE LAW, IN
+NO EVENT WILL LICENSOR BE LIABLE TO YOU ON ANY LEGAL THEORY FOR ANY SPECIAL,
+INCIDENTAL, CONSEQUENTIAL, PUNITIVE OR EXEMPLARY DAMAGES ARISING OUT OF THIS
+LICENSE OR THE USE OF THE WORK, EVEN IF LICENSOR HAS BEEN ADVISED OF THE
+POSSIBILITY OF SUCH DAMAGES.
+
+7. Termination
+
+ a. This License and the rights granted hereunder will terminate automatically
+ upon any breach by You of the terms of this License. Individuals or
+ entities who have received Derivative Works or Collective Works from You
+ under this License, however, will not have their licenses terminated
+ provided such individuals or entities remain in full compliance with those
+ licenses. Sections 1, 2, 5, 6, 7, and 8 will survive any termination of
+ this License.
+
+ b. Subject to the above terms and conditions, the license granted here is
+ perpetual (for the duration of the applicable copyright in the
+ Work). Notwithstanding the above, Licensor reserves the right to release
+ the Work under different license terms or to stop distributing the Work at
+ any time; provided, however that any such election will not serve to
+ withdraw this License (or any other license that has been, or is required
+ to be, granted under the terms of this License), and this License will
+ continue in full force and effect unless terminated as stated above.
+
+8. Miscellaneous
+
+ a. Each time You distribute or publicly digitally perform the Work (as defined
+ in Section 1 above) or a Collective Work (as defined in Section 1 above),
+ the Licensor offers to the recipient a license to the Work on the same
+ terms and conditions as the license granted to You under this License.
+
+ b. Each time You distribute or publicly digitally perform a Derivative Work,
+ Licensor offers to the recipient a license to the original Work on the same
+ terms and conditions as the license granted to You under this License.
+
+ c. If any provision of this License is invalid or unenforceable under
+ applicable law, it shall not affect the validity or enforceability of the
+ remainder of the terms of this License, and without further action by the
+ parties to this agreement, such provision shall be reformed to the minimum
+ extent necessary to make such provision valid and enforceable.
+
+ d. No term or provision of this License shall be deemed waived and no breach
+ consented to unless such waiver or consent shall be in writing and signed
+ by the party to be charged with such waiver or consent.
+
+ e. This License constitutes the entire agreement between the parties with
+ respect to the Work licensed here. There are no understandings, agreements
+ or representations with respect to the Work not specified here. Licensor
+ shall not be bound by any additional provisions that may appear in any
+ communication from You. This License may not be modified without the mutual
+ written agreement of the Licensor and You.
+
+Creative Commons Notice
+
+ Creative Commons is not a party to this License, and makes no warranty
+ whatsoever in connection with the Work. Creative Commons will not be liable
+ to You or any party on any legal theory for any damages whatsoever,
+ including without limitation any general, special, incidental or
+ consequential damages arising in connection to this
+ license. Notwithstanding the foregoing two (2) sentences, if Creative
+ Commons has expressly identified itself as the Licensor hereunder, it shall
+ have all rights and obligations of Licensor.
+
+ Except for the limited purpose of indicating to the public that the Work is
+ licensed under the CCPL, Creative Commons does not authorize the use by
+ either party of the trademark "Creative Commons" or any related trademark
+ or logo of Creative Commons without the prior written consent of Creative
+ Commons. Any permitted use will be in compliance with Creative Commons'
+ then-current trademark usage guidelines, as may be published on its website
+ or otherwise made available upon request from time to time. For the
+ avoidance of doubt, this trademark restriction does not form part of this
+ License.
+
+ Creative Commons may be contacted at http://creativecommons.org/.
gem 'zenweb'
gem 'liquid'
gem 'RedCloth'
-gem 'colorize'
\ No newline at end of file
+gem 'colorize'
--- /dev/null
+h1. Arvados documentation
+
+This is the source code for "doc.arvados.org":http://doc.arvados.org.
+
+Here's how to build the HTML pages locally so you can preview your updates before you commit and push.
+
+Additional information is available on the "'Documentation' page on the Arvados wiki":https://arvados.org/projects/arvados/wiki/Documentation.
+
+h2. Install dependencies
+
+<pre>
+arvados/doc$ bundle install
+</pre>
+
+h2. Generate HTML pages
+
+<pre>
+arvados/doc$ rake
+</pre>
+
+Alternately, to make the documentation browsable on the local filesystem:
+
+<pre>
+arvados/doc$ rake generate baseurl=$PWD/.site
+</pre>
+
+h2. Run linkchecker
+
+If you have "Linkchecker":http://wummel.github.io/linkchecker/ installed on
+your system, you can run it against the documentation:
+
+<pre>
+arvados/doc$ rake linkchecker baseurl=file://$PWD/.site
+</pre>
+
+Please note that this will regenerate your $PWD/.site directory.
+
+h2. Preview HTML pages
+
+<pre>
+arvados/doc$ rake run
+[2014-03-10 09:03:41] INFO WEBrick 1.3.1
+[2014-03-10 09:03:41] INFO ruby 2.1.1 (2014-02-24) [x86_64-linux]
+[2014-03-10 09:03:41] INFO WEBrick::HTTPServer#start: pid=8926 port=8000
+</pre>
+
+Preview the rendered pages at "http://localhost:8000":http://localhost:8000.
+
+h2. Publish HTML pages inside Workbench
+
+(or some other web site)
+
+You can set @baseurl@ (the URL prefix for all internal links), @arvados_api_host@ and @arvados_workbench_host@ without changing @_config.yml@:
+
+<pre>
+arvados/doc$ rake generate baseurl=/doc arvados_api_host=xyzzy.arvadosapi.com
+</pre>
+
+Make the docs appear at {workbench_host}/doc by creating a symbolic link in Workbench's @public@ directory, pointing to the generated HTML tree.
+
+<pre>
+arvados/doc$ ln -sn ../../../doc/.site ../apps/workbench/public/doc
+</pre>
+
+h2. Delete generated files
+
+<pre>
+arvados/doc$ rake realclean
+</pre>
require "rubygems"
require "colorize"
-require "zenweb/tasks"
-load "zenweb-textile.rb"
-load "zenweb-liquid.rb"
+task :generate => [ :realclean, 'sdk/python/arvados/index.html' ] do
+ vars = ['baseurl', 'arvados_api_host', 'arvados_workbench_host']
+ vars.each do |v|
+ if ENV[v]
+ website.config.h[v] = ENV[v]
+ end
+ end
+end
file "sdk/python/arvados/index.html" do |t|
`which epydoc`
if $? == 0
- `epydoc --html -o sdk/python/arvados arvados`
- Dir["sdk/python/arvados/*"].each do |f|
- puts f
- $website.pages[f] = Zenweb::Page.new($website, f)
- end
+ `epydoc --html --parse-only -o sdk/python/arvados ../sdk/python/arvados/`
else
puts "Warning: epydoc not found, Python documentation will not be generated".colorize(:light_red)
end
end
+task :linkchecker => [ :generate ] do
+ Dir.chdir(".site") do
+ `which linkchecker`
+ if $? == 0
+ system "linkchecker index.html --ignore-url='!file://'"
+ else
+ puts "Warning: linkchecker not found, skipping run".colorize(:light_red)
+ end
+ end
+end
+
+task :clean do
+ rm_rf "sdk/python/arvados"
+end
+
+require "zenweb/tasks"
+load "zenweb-textile.rb"
+load "zenweb-liquid.rb"
+
task :extra_wirings do
$website.pages["sdk/python/python.html.textile.liquid"].depends_on("sdk/python/arvados/index.html")
end
# file:///tmp/arvados/doc/.site). To make docs show up inside
# workbench, use /doc here and add a symlink at
# apps/workbench/public/doc pointing to ../../../doc/.site
+# You can also set these on the command line:
+# $ rake generate baseurl=/example arvados_api_host=example.comA
-baseurl: /doc
+baseurl:
+arvados_api_host: localhost
+arvados_workbench_host: localhost
exclude: ["Rakefile", "tmp", "vendor"]
-arvados_api_host: qr1hi.arvadosapi.com
navbar:
userguide:
- user/getting_started/community.html.textile.liquid
- Tutorials:
- user/tutorials/tutorial-keep.html.textile.liquid
- - user/tutorials/tutorial-job1.html.textile.liquid
+ - user/tutorials/intro-crunch.html.textile.liquid
+ - user/tutorials/tutorial-pipeline-workbench.html.textile.liquid
- user/tutorials/tutorial-firstscript.html.textile.liquid
- - user/tutorials/tutorial-job-debug.html.textile.liquid
- - user/tutorials/tutorial-parallel.html.textile.liquid
- user/tutorials/tutorial-new-pipeline.html.textile.liquid
- - user/tutorials/tutorial-trait-search.html.textile.liquid
- - user/tutorials/tutorial-gatk-variantfiltration.html.textile.liquid
- user/tutorials/running-external-program.html.textile.liquid
+ - Intermediate topics:
+ - user/topics/running-pipeline-command-line.html.textile.liquid
+ - user/topics/tutorial-job1.html.textile.liquid
+ - user/topics/tutorial-job-debug.html.textile.liquid
+ - user/topics/tutorial-parallel.html.textile.liquid
+ - user/topics/tutorial-trait-search.html.textile.liquid
+ - user/topics/tutorial-gatk-variantfiltration.html.textile.liquid
+ - user/topics/keep.html.textile.liquid
- Examples:
- user/examples/crunch-examples.html.textile.liquid
- Reference:
- user/reference/api-tokens.html.textile.liquid
- user/reference/sdk-cli.html.textile.liquid
+ - user/reference/job-and-pipeline-reference.html.textile.liquid
+ - Arvados License:
+ - user/copying/copying.html.textile.liquid
+ - user/copying/agpl-3.0.html
+ - user/copying/LICENSE-2.0.html
+ - user/copying/by-sa-3.0.html
sdk:
- Overview:
- sdk/index.html.textile.liquid
- sdk/python/sdk-python.html.textile.liquid
- sdk/python/python.html.textile.liquid
- sdk/python/crunch-utility-libraries.html.textile.liquid
+ - Perl:
+ - sdk/perl/index.html.textile.liquid
+ - Ruby:
+ - sdk/ruby/index.html.textile.liquid
+ - CLI:
+ - sdk/cli/index.html.textile.liquid
api:
- Concepts:
- api/index.html.textile.liquid
- admin/cheat_sheet.html.textile.liquid
installguide:
- Install:
- - install/install-api-server.html.md.liquid
- install/index.html.md.liquid
- - install/install-workbench-app.html.md.liquid
+ - install/install-sso.html.textile.liquid
+ - install/install-api-server.html.textile.liquid
+ - install/install-workbench-app.html.textile.liquid
- install/client.html.textile.liquid
- install/create-standard-objects.html.textile.liquid
- install/install-crunch-dispatch.html.textile.liquid
import arvados
-arvados.job_setup.one_task_per_input_file(if_sequence=0, and_end_task=True)
-this_task = arvados.current_task()
+# Automatically parallelize this job by running one task per file.
+arvados.job_setup.one_task_per_input_file(if_sequence=0, and_end_task=True, input_as_path=True)
-# Get the input collection for this task
-this_task_input = this_task['parameters']['input']
+# Get the input file for the task
+input_file = arvados.get_task_param_mount('input')
-# Create a CollectionReader to access the collection
-input_collection = arvados.CollectionReader(this_task_input)
+# Run the external 'md5sum' program on the input file
+stdoutdata, stderrdata = arvados.util.run_command(['md5sum', input_file])
-# Get the name of the first file in the collection
-input_file = list(input_collection.all_files())[0].name()
-
-# Extract the file to a temporary directory
-# Returns the directory that the file was written to
-input_dir = arvados.util.collection_extract(this_task_input,
- 'tmp',
- files=[input_file],
- decompress=False)
-
-# Run the external 'md5sum' program on the input file, with the current working
-# directory set to the location the input file was extracted to.
-stdoutdata, stderrdata = arvados.util.run_command(
- ['md5sum', input_file],
- cwd=input_dir)
-
-# Save the standard output (stdoutdata) "md5sum.txt" in the output collection
+# Save the standard output (stdoutdata) to "md5sum.txt" in the output collection
out = arvados.CollectionWriter()
out.set_current_file_name("md5sum.txt")
out.write(stdoutdata)
-
-this_task.set_output(out.finish())
+arvados.current_task().set_output(out.finish())
#!/usr/bin/env python
-# Import the hashlib module (part of the Python standard library) to compute md5.
-import hashlib
+import hashlib # Import the hashlib module to compute md5.
+import arvados # Import the Arvados sdk module
-# Import the Arvados sdk module
-import arvados
+# Automatically parallelize this job by running one task per file.
+# This means that if the input consists of many files, each file will
+# be processed in parallel on different nodes enabling the job to
+# be completed quicker.
+arvados.job_setup.one_task_per_input_file(if_sequence=0, and_end_task=True,
+ input_as_path=True)
-# Get information about the task from the environment
-this_task = arvados.current_task()
-
-# Get the "input" field from "script_parameters" on the job creation object
-this_job_input = arvados.getjobparam('input')
-
-# Create the object access to the collection referred to in the input
-collection = arvados.CollectionReader(this_job_input)
-
-# Create an object to write a new collection as output
-out = arvados.CollectionWriter()
-
-# Set the name of output file within the collection
-out.set_current_file_name("md5sum.txt")
+# Create the message digest object that will compute the md5 hash
+digestor = hashlib.new('md5')
-# Get an iterator over the files listed in the collection
-all_files = collection.all_files()
-
-# Iterate over each file
-for input_file in all_files:
- # Create the object that will actually compute the md5 hash
- digestor = hashlib.new('md5')
+# Get the input file for the task
+input_file = arvados.get_task_param_mount('input')
+# Open the input file for reading
+with open(input_file) as f:
while True:
- # read a 1 megabyte block from the file
- buf = input_file.read(2**20)
-
- # break when there is no more data left
- if len(buf) == 0:
+ buf = f.read(2**20) # read a 1 megabyte block from the file
+ if len(buf) == 0: # break when there is no more data left
break
+ digestor.update(buf) # update the md5 hash object
- # update the md5 hash object
- digestor.update(buf)
-
- # Get the final hash code
- hexdigest = digestor.hexdigest()
+# Get object representing the current task
+this_task = arvados.current_task()
- # Get the file name from the StreamFileReader object
- file_name = input_file.name()
+ # Write a new collection as output
+out = arvados.CollectionWriter()
- # The "stream name" is the subdirectory inside the collection in which
- # the file is located; '.' is the root of the collection.
- if input_file.stream_name() != '.':
- file_name = os.join(input_file.stream_name(), file_name)
+ # Set output file within the collection
+out.set_current_file_name("md5sum.txt")
- # Write an output line with the md5 value and file name.
- out.write("%s %s\n" % (hexdigest, file_name))
+# Write an output line with the md5 value and input
+out.write("%s %s\n" % (digestor.hexdigest(), this_task['parameters']['input']))
-# Commit the output to keep. This returns a Keep id.
+ # Commit the output to keep. This returns a Keep id.
output_id = out.finish()
# Set the output for this task to the Keep id
-this_task.set_output(output_id)
+this_task.set_output(output_id)
# Done!
--- /dev/null
+{% assign n = 0 %}
+{% assign prev = "" %}
+{% assign nx = 0 %}
+{% for section in site.navbar[page.navsection] %}
+ {% for entry in section %}
+ {% for item in entry[1] %}
+ {% assign p = site.pages[item] %}
+ {% if nx == 1 %}
+ <hr>
+ {% if prev != "" %}
+ <a href="{{ site.baseurl }}{{ prev.url }}" class="pull-left">Previous: {{ prev.title }}</a>
+ {% endif %}
+ <a href="{{ site.baseurl }}{{ p.url }}" class="pull-right">Next: {{ p.title }}</a>
+ {% assign nx = 0 %}
+ {% assign n = 1 %}
+ {% endif %}
+ {% if p.url == page.url %}
+ {% assign nx = 1 %}
+ {% else %}
+ {% assign prev = p %}
+ {% endif %}
+ {% endfor %}
+ {% endfor %}
+{% endfor %}
+{% if n == 0 && prev != "" %}
+ <hr>
+ <a href="{{ site.baseurl }}{{ prev.url }}" class="pull-left">Previous: {{ prev.title }}</a>
+ {% assign n = 1 %}
+{% endif %}
\ No newline at end of file
<link href="{{ site.baseurl }}/css/bootstrap.css" rel="stylesheet">
<link href="{{ site.baseurl }}/css/nav-list.css" rel="stylesheet">
<link href="{{ site.baseurl }}/css/badges.css" rel="stylesheet">
+ <link href="{{ site.baseurl }}/css/code.css" rel="stylesheet">
<style>
html {
height:100%;
text-align: center;
margin-bottom: 1em;
}
- .userinput {
- color: #d14;
- }
:target {
padding-top: 61px;
margin-top: -61px;
}
</style>
- <link href="{{ site.baseurl }}/css/bootstrap-responsive.min.css" rel="stylesheet">
+
<!-- HTML5 shim, for IE6-8 support of HTML5 elements -->
<!--[if lt IE 9]>
<script src="../assets/js/html5shiv.js"></script>
<div class="row">
{% include 'navbar_left' %}
<div class="col-sm-9">
+ <h1>{{ page.title }}</h1>
{{ content }}
+ {% include 'webring' %}
</div>
</div>
</script>
-<p style="text-align: center">
-The content of this documentation is licensed under a
-<a href="https://creativecommons.org/licenses/by-sa/3.0/us/">Creative
- Commons Attribution-Share Alike 3.0 United States</a> licence.
+<p style="text-align: center"><small>
+The content of this documentation is licensed under the
+<a href="{{ site.baseurl }}/user/copying/by-sa-3.0.html">Creative
+ Commons Attribution-Share Alike 3.0 United States</a> licence.<br>
+Code samples in this documentation are licensed under the
+<a href="{{ site.baseurl }}/user/copying/LICENSE-2.0.html">Apache License, Version 2.0.</a></small>
</p>
</body>
title: Cheat Sheet
...
-h1. Cheat Sheet
+
h3. CLI setup
read -rd $'\000' newlink <<EOF; arv link create --link "$newlink"
{
-"tail_kind":"arvados#user",
"tail_uuid":"$user_uuid",
-"head_kind":"arvados#virtualMachine",
"head_uuid":"$vm_uuid",
"link_class":"permission",
"name":"can_login",
read -rd $'\000' newlink <<EOF; arv link create --link "$newlink"
{
-"tail_kind":"arvados#user",
"tail_uuid":"$user_uuid",
-"head_kind":"arvados#repository",
"head_uuid":"$repo_uuid",
"link_class":"permission",
"name":"can_write",
...
-h1. Authentication
+
Every API request (except the authentication API itself) includes an @access_token@ parameter.
...
-h1. API Reference
+
h2. Concepts
...
-h1. REST Methods
+
(using Group as an example)
...
-h1. api_client_authorizations
+
Required arguments are displayed in %{background:#ccffcc}green%.
...
-h1. api_clients
+
Required arguments are displayed in %{background:#ccffcc}green%.
...
-h1. authorized_keys
+
Required arguments are displayed in %{background:#ccffcc}green%.
...
-h1. collections
+
Required arguments are displayed in %{background:#ccffcc}green%.
...
-h1. groups
+
Required arguments are displayed in %{background:#ccffcc}green%.
...
-h1. humans
+
Required arguments are displayed in %{background:#ccffcc}green%.
...
-h1. job_tasks
+
Required arguments are displayed in %{background:#ccffcc}green%.
...
-h1. jobs
+
Required arguments are displayed in %{background:#ccffcc}green%.
...
-h1. keep_disks
+
Required arguments are displayed in %{background:#ccffcc}green%.
...
-h1. links
+
Required arguments are displayed in %{background:#ccffcc}green%.
...
-h1. logs
+
Required arguments are displayed in %{background:#ccffcc}green%.
...
-h1. nodes
+
Required arguments are displayed in %{background:#ccffcc}green%.
...
-h1. pipeline_instances
+
Required arguments are displayed in %{background:#ccffcc}green%.
...
-h1. pipeline_templates
+
Required arguments are displayed in %{background:#ccffcc}green%.
...
-h1. repositories
+
Required arguments are displayed in %{background:#ccffcc}green%.
...
-h1. specimens
+
Required arguments are displayed in %{background:#ccffcc}green%.
...
-h1. traits
+
Required arguments are displayed in %{background:#ccffcc}green%.
...
-h1. users
+
Required arguments are displayed in %{background:#ccffcc}green%.
...
-h1. virtual_machines
+
Required arguments are displayed in %{background:#ccffcc}green%.
...
-h1. Permission model
+
Each API transaction (read, write, create, etc.) is done on behalf of a person.
A user (person) is permitted to act on an object if there is a path (series of permission Links) from the acting user to the object in which
-* Every intervening object is a Group, and
+* Every intervening object is a Group or a User, and
* Every intervening permission Link allows the current action
Each object has exactly one _owner_, which can be either a User or a Group.
Three lab members are working together on a project. All Specimens, Links, Jobs, etc. can be modified by any of the three lab members. _Other_ lab members, who are not working on this project, can view but not modify these objects.
-h3. 4. Segregated roles
+h3. 4. Group-level administrator
+
+The Ashton Lab administrator, Alison, manages user accounts within her lab. She can enable and disable accounts, and exercise any permission that her lab members have.
+
+George has read-only access to the same set of accounts. This lets him see things like user activity and resource usage reports, without worrying about accidentally messing up anyone's data.
+
+table(table table-bordered table-condensed).
+|Tail |Permission |Head |Effect|
+|Group: Ashton Lab Admin|can_manage |User: Lab Member 1 |Lab member 1 is in this administrative group|
+|Group: Ashton Lab Admin|can_manage |User: Lab Member 2 |Lab member 2 is in this administrative group|
+|Group: Ashton Lab Admin|can_manage |User: Lab Member 3 |Lab member 3 is in this administrative group|
+|Group: Ashton Lab Admin|can_manage |User: Alison |Alison is in this administrative group|
+|Group: Ashton Lab Admin|can_manage |User: George |George is in this administrative group|
+|Alison |can_manage |Group: Ashton Lab Admin |Alison can do everything the above lab members can do|
+|George |can_read |Group: Ashton Lab Admin |George can read everything the above lab members can read|
+
+h3. 5. Segregated roles
Granwyth, at the Hulatberi Lab, sets up a Factory Robot which uses a hosted Arvados site to do work for the Hulatberi Lab.
...
-h1. Resources
+
This page describes the common attributes of Arvados resources.
...
-h1. ApiClient
+
An **ApiClient** represents a client program that has issued a request to the API server.
h2. Methods
-See "REST methods for working with Arvados resources":/api/methods.html
+See "REST methods for working with Arvados resources":{{site.baseurl}}/api/methods.html
API endpoint base: @https://{{ site.arvados_api_host }}/arvados/v1/api_clients@
h2. Resources
-Each ApiClient has, in addition to the usual "attributes of Arvados resources":/api/resources.html:
+Each ApiClient has, in addition to the usual "attributes of Arvados resources":{{site.baseurl}}/api/resources.html:
table(table table-bordered table-condensed).
|_. Attribute|_. Type|_. Description|_. Example|
...
-h1. ApiClientAuthorization
+
A **ApiClientAuthorization** represents the API authorization token that has been issued to each "ApiClient":ApiClient.html known to this Arvados instance.
h2. Methods
-See "REST methods for working with Arvados resources":/api/methods.html
+See "REST methods for working with Arvados resources":{{site.baseurl}}/api/methods.html
-API endpoint base: @https://{{ site.arvados_api_host }}/arvados/v1/api_client_authorization@
+API endpoint base: @https://{{ site.arvados_api_host }}/arvados/v1/api_client_authorizations@
h2. Creation
navsection: api
navmenu: Schema
title: AuthorizedKey
-
...
-h1. AuthorizedKey
+
A **AuthorizedKey** represents...
h2. Methods
-See "REST methods for working with Arvados resources":/api/methods.html
+See "REST methods for working with Arvados resources":{{site.baseurl}}/api/methods.html
-API endpoint base: @https://{{ site.arvados_api_host }}/arvados/v1/authorized_key@
+API endpoint base: @https://{{ site.arvados_api_host }}/arvados/v1/authorized_keys@
h2. Creation
h2. Resources
-Each AuthorizedKey has, in addition to the usual "attributes of Arvados resources":/api/resources.html:
+Each AuthorizedKey has, in addition to the usual "attributes of Arvados resources":{{site.baseurl}}/api/resources.html:
table(table table-bordered table-condensed).
|_. Attribute|_. Type|_. Description|_. Example|
...
-h1. Collection
-This resource concerns metadata, usage accounting, and integrity checks for data stored on the cloud. Reading and writing the data _per se_ is achieved by the "Keep":/user/tutorials/tutorial-keep.html storage system.
+
+This resource concerns metadata, usage accounting, and integrity checks for data stored on the cloud. Reading and writing the data _per se_ is achieved by the "Keep":{{site.baseurl}}/user/tutorials/tutorial-keep.html storage system.
h2. Methods
-See "REST methods for working with Arvados resources":/api/methods.html
+See "REST methods for working with Arvados resources":{{site.baseurl}}/api/methods.html
API endpoint base: @https://{{ site.arvados_api_host }}/arvados/v1/links@
h2. Resource
-Each collection has, in addition to the usual "attributes of Arvados resources":/api/resources.html:
+Each collection has, in addition to the usual "attributes of Arvados resources":{{site.baseurl}}/api/resources.html:
table(table table-bordered table-condensed).
|_. Attribute|_. Type|_. Description|_. Example|
...
-h1. Commit
+
A **Commit** represents...
h2. Methods
-See "REST methods for working with Arvados resources":/api/methods.html
+See "REST methods for working with Arvados resources":{{site.baseurl}}/api/methods.html
-API endpoint base: @https://{{ site.arvados_api_host }}/arvados/v1/commit@
+API endpoint base: @https://{{ site.arvados_api_host }}/arvados/v1/commits@
h2. Creation
h2. Resources
-Each Commit has, in addition to the usual "attributes of Arvados resources":/api/resources.html:
+Each Commit has, in addition to the usual "attributes of Arvados resources":{{site.baseurl}}/api/resources.html:
table(table table-bordered table-condensed).
|_. Attribute|_. Type|_. Description|_. Example|
...
-h1. CommitAncestor
+
A **CommitAncestor** represents...
h2. Methods
-See "REST methods for working with Arvados resources":/api/methods.html
+See "REST methods for working with Arvados resources":{{site.baseurl}}/api/methods.html
-API endpoint base: @https://{{ site.arvados_api_host }}/arvados/v1/commit_ancestor@
+API endpoint base: @https://{{ site.arvados_api_host }}/arvados/v1/commit_ancestors@
h2. Creation
h2. Resources
-Each CommitAncestor has, in addition to the usual "attributes of Arvados resources":/api/resources.html:
+Each CommitAncestor has, in addition to the usual "attributes of Arvados resources":{{site.baseurl}}/api/resources.html:
table(table table-bordered table-condensed).
|_. Attribute|_. Type|_. Description|_. Example|
...
-h1. Group
+
A **Group** represents...
h2. Methods
-See "REST methods for working with Arvados resources":/api/methods.html
+See "REST methods for working with Arvados resources":{{site.baseurl}}/api/methods.html
-API endpoint base: @https://{{ site.arvados_api_host }}/arvados/v1/group@
+API endpoint base: @https://{{ site.arvados_api_host }}/arvados/v1/groups@
h2. Creation
h2. Resources
-Each Group has, in addition to the usual "attributes of Arvados resources":/api/resources.html:
+Each Group has, in addition to the usual "attributes of Arvados resources":{{site.baseurl}}/api/resources.html:
table(table table-bordered table-condensed).
|_. Attribute|_. Type|_. Description|_. Example|
...
-h1. Human
+
A **Human** represents...
h2. Methods
-See "REST methods for working with Arvados resources":/api/methods.html
+See "REST methods for working with Arvados resources":{{site.baseurl}}/api/methods.html
-API endpoint base: @https://{{ site.arvados_api_host }}/arvados/v1/human@
+API endpoint base: @https://{{ site.arvados_api_host }}/arvados/v1/humans@
h2. Creation
h2. Resources
-Each Human has, in addition to the usual "attributes of Arvados resources":/api/resources.html:
+Each Human has, in addition to the usual "attributes of Arvados resources":{{site.baseurl}}/api/resources.html:
table(table table-bordered table-condensed).
|_. Attribute|_. Type|_. Description|_. Example|
...
-h1. Job
+
Applications submit compute jobs when:
* Provenance is important, i.e., it is worth recording how the output was produced; or
h2. Methods
-See "REST methods for working with Arvados resources":/api/methods.html
+See "REST methods for working with Arvados resources":{{site.baseurl}}/api/methods.html
API endpoint base: @https://{{ site.arvados_api_host }}/arvados/v1/jobs@
h3. Additional parameters for "Create" method
table(table table-bordered table-condensed).
-|_. Parameter name|_. Type|_. Description|
-|allow_duplicate|boolean|If true, a new job is submitted even if an identical job has already been submitted (and has not failed). If false or not supplied, a new job will _not_ be submitted and the existing job will be returned in the API response.|
+|_. Attribute |_. Type|_. Accepted values |_. Required|_. Description|
+|script |string |filename |yes |The actual script that will be run by crunch. Must be the name of an executable file in the crunch_scripts/ directory at the git revision specified by script_version.|
+|script_version |string |git branch, tag, or version hash |yes |The code version to run, which is available in the specified repository. May be a git hash or tag to specify an exact version, or a branch. If it is a branch, use the branch head.|
+|repository |string |name of git repository hosted by Arvados |yes |The repository to search for script_version.|
+|script_parameters |object |any JSON object |yes |The input parameters for the job, with the parameter names as keys mapping to parameter values.|
+|minimum_script_version |string |git branch, tag, or version hash |no |The minimum acceptable script version when deciding whether to re-use a past job.|
+|exclude_script_versions|array of strings|git branch, tag, or version hash|no |Script versions to exclude when deciding whether to re-use a past job.|
+|nondeterministic |boolean | |no |If true, never re-use a past job, and flag this job so it will never be considered for re-use.|
+|no_reuse |boolean | |no |If true, do not re-use a past job, but this job may be re-used.|
+
+See the "job and pipeline reference":{{site.baseurl}}/user/reference/job-and-pipeline-reference.html for more information.
h3. Queue
<pre>
GET https://{{ site.arvados_api_host }}/arvados/v1/jobs/queue
-
+
POST https://{{ site.arvados_api_host }}/arvados/v1/jobs/queue
_method=GET
where[owner_uuid]=xyzzy-tpzed-a4lcehql0dv2u25
→ Job resource list
-This method is equivalent to the "index method":/api/methods.html#index, except that the results are restricted to queued jobs (i.e., jobs that have not yet been started or cancelled) and order defaults to queue priority.
+This method is equivalent to the "index method":{{site.baseurl}}/api/methods.html#index, except that the results are restricted to queued jobs (i.e., jobs that have not yet been started or cancelled) and order defaults to queue priority.
h2. Resource
-Each job has, in addition to the usual "attributes of Arvados resources":/api/resources.html:
+Each job has, in addition to the usual "attributes of Arvados resources":{{site.baseurl}}/api/resources.html:
table(table table-bordered table-condensed).
|_. Attribute|_. Type|_. Description|_. Notes|
|runtime_constraints{}|list|Constraints that must be satisfied by the job/task scheduler in order to run the job.|See below.|
|script|string|Name of crunch script in @/crunch_scripts/@||
|script_parameters{}|list|Parameters passed to MapReduce script||
-|script_version|string|git commit/tree used when running the job|This can be submitted as an unambiguous prefix of a commit sha1, "repository:tag", or "repository:branch". Before the job starts, Arvados will resolve it to a full 40-character git commit sha1.|
+|script_version|string|The git ref of the the git commit used to run the job.| When the job starts, Arvados updates this field to the precise git commit hash used by the job.|
+|repository |string |The repository from which the script_version will be fetched.|
|cancelled_by_client_uuid|string|API client ID|Is null if job has not been cancelled|
|cancelled_by_user_uuid|string|Authenticated user ID|Is null if job has not been cancelled|
|cancelled_at|datetime|When job was cancelled|Is null if job has not been cancelled|
|log|string|||
|tasks_summary|Hash|||
|output|string|||
+|nondeterministic |boolean |Indicates whether this job is a candidate for re-use|If true, this job will never be considered for re-use.|
h3. Runtime constraints
...
-h1. JobTask
+
A Job Task is a well defined independently-computable portion of a "Job":Job.html.
h2. Methods
-See "REST methods for working with Arvados resources":/api/methods.html
+See "REST methods for working with Arvados resources":{{site.baseurl}}/api/methods.html
API endpoint base: @https://{{ site.arvados_api_host }}/arvados/v1/job_tasks@
h2. Resources
-Each JobTask has, in addition to the usual "attributes of Arvados resources":/api/resources.html:
+Each JobTask has, in addition to the usual "attributes of Arvados resources":{{site.baseurl}}/api/resources.html:
table(table table-bordered table-condensed).
|_. Attribute|_. Type|_. Description|_. Example|
...
-h1. KeepDisk
+
A **KeepDisk** represents...
h2. Methods
-See "REST methods for working with Arvados resources":/api/methods.html
+See "REST methods for working with Arvados resources":{{site.baseurl}}/api/methods.html
-API endpoint base: @https://{{ site.arvados_api_host }}/arvados/v1/keep_disk@
+API endpoint base: @https://{{ site.arvados_api_host }}/arvados/v1/keep_disks@
h2. Creation
h2. Resources
-Each KeepDisk has, in addition to the usual "attributes of Arvados resources":/api/resources.html:
+Each KeepDisk has, in addition to the usual "attributes of Arvados resources":{{site.baseurl}}/api/resources.html:
table(table table-bordered table-condensed).
|_. Attribute|_. Type|_. Description|_. Example|
...
-h1. Link
+
**Links** describe relationships between Arvados objects, and from objects to primitives.
h2. Methods
-See "REST methods for working with Arvados resources":/api/methods.html
+See "REST methods for working with Arvados resources":{{site.baseurl}}/api/methods.html
API endpoint base: @https://{{ site.arvados_api_host }}/arvados/v1/links@
h2. Resource
-Each link has, in addition to the usual "attributes of Arvados resources":/api/resources.html:
+Each link has, in addition to the usual "attributes of Arvados resources":{{site.baseurl}}/api/resources.html:
table(table table-bordered table-condensed).
|_. Attribute|_. Type|_. Description|
|tail_uuid|string|Object UUID at the tail (start, source, origin) of this link|
-|tail_kind|string|Object kind at the tail (start, source, origin) of this link|
|link_class|string|Class (see below)|
|name|string|Link type (see below)|
|head_uuid|string|Object UUID at the head (end, destination, target) of this link|
-|head_kind|string|Object kind at the head (end, destination, target) of this link|
|properties{}|list|Additional information, expressed as a key→value hash. Key: string. Value: string, number, array, or hash.|
h2. Link classes
...
-h1. Log
+
**Log** objects record events that occur in an Arvados cluster. Both user-written pipelines and the Arvados system itself may generate Log events.
h2. Methods
-See "REST methods for working with Arvados resources":/api/methods.html
+See "REST methods for working with Arvados resources":{{site.baseurl}}/api/methods.html
-API endpoint base: @https://{{ site.arvados_api_host }}/arvados/v1/log@
+API endpoint base: @https://{{ site.arvados_api_host }}/arvados/v1/logs@
h2. Creation
h2. Resources
-Each Log has, in addition to the usual "attributes of Arvados resources":/api/resources.html:
+Each Log has, in addition to the usual "attributes of Arvados resources":{{site.baseurl}}/api/resources.html:
table(table table-bordered table-condensed).
|_. Attribute|_. Type|_. Description|_. Example|
-|object_kind|string|||
|object_uuid|string|||
|event_at|datetime|||
|event_type|string|A user-defined category or type for this event.|@LOGIN@|
...
-h1. Node
+
A **Node** represents...
h2. Methods
-See "REST methods for working with Arvados resources":/api/methods.html
+See "REST methods for working with Arvados resources":{{site.baseurl}}/api/methods.html
-API endpoint base: @https://{{ site.arvados_api_host }}/arvados/v1/node@
+API endpoint base: @https://{{ site.arvados_api_host }}/arvados/v1/nodes@
h2. Creation
h2. Resources
-Each Node has, in addition to the usual "attributes of Arvados resources":/api/resources.html:
+Each Node has, in addition to the usual "attributes of Arvados resources":{{site.baseurl}}/api/resources.html:
table(table table-bordered table-condensed).
|_. Attribute|_. Type|_. Description|_. Example|
...
-h1. PipelineInstance
+
A **PipelineInstance** represents...
h2. Methods
-See "REST methods for working with Arvados resources":/api/methods.html
+See "REST methods for working with Arvados resources":{{site.baseurl}}/api/methods.html
-API endpoint base: @https://{{ site.arvados_api_host }}/arvados/v1/pipeline_instance@
+API endpoint base: @https://{{ site.arvados_api_host }}/arvados/v1/pipeline_instances@
h2. Creation
h2. Resources
-Each PipelineInstance has, in addition to the usual "attributes of Arvados resources":/api/resources.html:
+Each PipelineInstance has, in addition to the usual "attributes of Arvados resources":{{site.baseurl}}/api/resources.html:
table(table table-bordered table-condensed).
|_. Attribute|_. Type|_. Description|_. Example|
...
-h1. PipelineTemplate
+
A **PipelineTemplate** represents...
h2. Methods
-See "REST methods for working with Arvados resources":/api/methods.html
+See "REST methods for working with Arvados resources":{{site.baseurl}}/api/methods.html
-API endpoint base: @https://{{ site.arvados_api_host }}/arvados/v1/pipeline_template@
+API endpoint base: @https://{{ site.arvados_api_host }}/arvados/v1/pipeline_templates@
h2. Creation
h2. Resources
-Each PipelineTemplate has, in addition to the usual "attributes of Arvados resources":/api/resources.html:
+Each PipelineTemplate has, in addition to the usual "attributes of Arvados resources":{{site.baseurl}}/api/resources.html:
table(table table-bordered table-condensed).
|_. Attribute|_. Type|_. Description|_. Example|
...
-h1. Repository
+
A **Repository** represents...
h2. Methods
-See "REST methods for working with Arvados resources":/api/methods.html
+See "REST methods for working with Arvados resources":{{site.baseurl}}/api/methods.html
-API endpoint base: @https://{{ site.arvados_api_host }}/arvados/v1/repository@
+API endpoint base: @https://{{ site.arvados_api_host }}/arvados/v1/repositories@
h2. Creation
h2. Resources
-Each Repository has, in addition to the usual "attributes of Arvados resources":/api/resources.html:
+Each Repository has, in addition to the usual "attributes of Arvados resources":{{site.baseurl}}/api/resources.html:
table(table table-bordered table-condensed).
|_. Attribute|_. Type|_. Description|_. Example|
...
-h1. Specimen
+
A **Specimen** represents...
h2. Methods
-See "REST methods for working with Arvados resources":/api/methods.html
+See "REST methods for working with Arvados resources":{{site.baseurl}}/api/methods.html
-API endpoint base: @https://{{ site.arvados_api_host }}/arvados/v1/specimen@
+API endpoint base: @https://{{ site.arvados_api_host }}/arvados/v1/specimens@
h2. Creation
h2. Resources
-Each Specimen has, in addition to the usual "attributes of Arvados resources":/api/resources.html:
+Each Specimen has, in addition to the usual "attributes of Arvados resources":{{site.baseurl}}/api/resources.html:
table(table table-bordered table-condensed).
|_. Attribute|_. Type|_. Description|_. Example|
...
-h1. Trait
+
A **Trait** represents...
h2. Methods
-See "REST methods for working with Arvados resources":/api/methods.html
+See "REST methods for working with Arvados resources":{{site.baseurl}}/api/methods.html
-API endpoint base: @https://{{ site.arvados_api_host }}/arvados/v1/trait@
+API endpoint base: @https://{{ site.arvados_api_host }}/arvados/v1/traits@
h2. Creation
h2. Resources
-Each Trait has, in addition to the usual "attributes of Arvados resources":/api/resources.html:
+Each Trait has, in addition to the usual "attributes of Arvados resources":{{site.baseurl}}/api/resources.html:
table(table table-bordered table-condensed).
|_. Attribute|_. Type|_. Description|_. Example|
...
-h1. User
+
A **User** represents...
h2. Methods
-See "REST methods for working with Arvados resources":/api/methods.html
+See "REST methods for working with Arvados resources":{{site.baseurl}}/api/methods.html
-API endpoint base: @https://{{ site.arvados_api_host }}/arvados/v1/user@
+API endpoint base: @https://{{ site.arvados_api_host }}/arvados/v1/users@
h2. Creation
h2. Resources
-Each User has, in addition to the usual "attributes of Arvados resources":/api/resources.html:
+Each User has, in addition to the usual "attributes of Arvados resources":{{site.baseurl}}/api/resources.html:
table(table table-bordered table-condensed).
|_. Attribute|_. Type|_. Description|_. Example|
...
-h1. VirtualMachine
+
A **VirtualMachine** represents...
h2. Methods
-See "REST methods for working with Arvados resources":/api/methods.html
+See "REST methods for working with Arvados resources":{{site.baseurl}}/api/methods.html
-API endpoint base: @https://{{ site.arvados_api_host }}/arvados/v1/virtual_machine@
+API endpoint base: @https://{{ site.arvados_api_host }}/arvados/v1/virtual_machines@
h2. Creation
h2. Resources
-Each VirtualMachine has, in addition to the usual "attributes of Arvados resources":/api/resources.html:
+Each VirtualMachine has, in addition to the usual "attributes of Arvados resources":{{site.baseurl}}/api/resources.html:
table(table table-bordered table-condensed).
|_. Attribute|_. Type|_. Description|_. Example|
--- /dev/null
+table.code {
+ font-family: Menlo,Monaco,Consolas,"Courier New",monospace;
+ display: block;
+ padding: 9.5px;
+ margin: 0px 0px 10px;
+ font-size: 13px;
+ line-height: 1.42857;
+ color: rgb(51, 51, 51);
+ word-break: break-all;
+ word-wrap: break-word;
+ background-color: rgb(245, 245, 245);
+ border: 1px solid rgb(204, 204, 204);
+ border-radius: 4px 4px 4px 4px;
+}
+
+table.code tr td {
+ white-space: pre;
+}
+
+table.code tr td:nth-child(2) {
+ color: #d14;
+ padding-left: .5em;
+}
+
+.userinput {
+ color: #d14;
+}
text-shadow: 0 -1px 0 rgba(0,0,0,.2);
background-color: rgb(66, 139, 202);
}
+
+.spaced-out li {
+ padding-bottom: 1em;
+}
\ No newline at end of file
"name":"GATK / exome PE fastq to snp",
"components":{
"extract-reference":{
+ "repository":"arvados",
+ "script_version":"e820bd1c6890f93ea1a84ffd5730bbf0e3d8e153",
"script":"file-select",
"script_parameters":{
"names":[
- "human_g1k_v37.fasta.gz", "human_g1k_v37.fasta.fai.gz", "human_g1k_v37.dict.gz"
+ "human_g1k_v37.fasta.gz",
+ "human_g1k_v37.fasta.fai.gz",
+ "human_g1k_v37.dict.gz"
],
"input":"d237a90bae3870b3b033aea1e99de4a9+10820+K@qr1hi"
},
- "script_version":"82a471c92036198aaf02ca0467ea48d49dbe822d"
+ "output_is_persistent":false
},
"bwa-index":{
- "script_version":"82a471c92036198aaf02ca0467ea48d49dbe822d",
+ "repository":"arvados",
+ "script_version":"e820bd1c6890f93ea1a84ffd5730bbf0e3d8e153",
"script":"bwa-index",
"script_parameters":{
"input":{
"output_of":"extract-reference"
},
"bwa_tbz":{
- "optional":false
+ "value":"8b6e2c4916133e1d859c9e812861ce13+70",
+ "required":true
}
- }
+ },
+ "output_is_persistent":false
},
"bwa-aln":{
- "script_version":"82a471c92036198aaf02ca0467ea48d49dbe822d",
+ "repository":"arvados",
+ "script_version":"e820bd1c6890f93ea1a84ffd5730bbf0e3d8e153",
"script":"bwa-aln",
"script_parameters":{
"input":{
- "optional":"false"
+ "dataclass":"Collection",
+ "required":"true"
},
"reference_index":{
"output_of":"bwa-index"
},
"samtools_tgz":{
- "optional":false
+ "value":"c777e23cf13e5d5906abfdc08d84bfdb+74",
+ "required":true
},
"bwa_tbz":{
- "optional":false
+ "value":"8b6e2c4916133e1d859c9e812861ce13+70",
+ "required":true
}
},
"runtime_constraints":{
"max_tasks_per_node":1
- }
+ },
+ "output_is_persistent":false
},
"picard-gatk2-prep":{
- "script_version":"82a471c92036198aaf02ca0467ea48d49dbe822d",
+ "repository":"arvados",
+ "script_version":"e820bd1c6890f93ea1a84ffd5730bbf0e3d8e153",
"script":"picard-gatk2-prep",
"script_parameters":{
"input":{
- "output_of":"bwa-aln"
+ "output_of":"bwa-aln"
},
"reference":{
"output_of":"extract-reference"
},
"picard_zip":{
- "optional":false
+ "value":"687f74675c6a0e925dec619cc2bec25f+77",
+ "required":true
}
},
"runtime_constraints":{
"max_tasks_per_node":1
- }
+ },
+ "output_is_persistent":false
},
"GATK2-realign":{
- "script_version":"82a471c92036198aaf02ca0467ea48d49dbe822d",
+ "repository":"arvados",
+ "script_version":"e820bd1c6890f93ea1a84ffd5730bbf0e3d8e153",
"script":"GATK2-realign",
"script_parameters":{
"input":{
- "output_of":"picard-gatk2-prep"
+ "output_of":"picard-gatk2-prep"
},
"gatk_bundle":{
- "optional":false
+ "value":"d237a90bae3870b3b033aea1e99de4a9+10820+K@qr1hi",
+ "required":true
},
"picard_zip":{
- "optional":false
+ "value":"687f74675c6a0e925dec619cc2bec25f+77",
+ "required":true
},
"gatk_tbz":{
- "optional":false
+ "value":"7e0a277d6d2353678a11f56bab3b13f2+87",
+ "required":true
},
"regions":{
- "optional":true
+ "value":"13b53dbe1ec032dfc495fd974aa5dd4a+87/S02972011_Covered_sort_merged.bed"
},
"region_padding":{
- "optional":true
+ "value":10
}
},
"runtime_constraints":{
"max_tasks_per_node":2
- }
+ },
+ "output_is_persistent":false
},
"GATK2-bqsr":{
- "script_version":"82a471c92036198aaf02ca0467ea48d49dbe822d",
+ "repository":"arvados",
+ "script_version":"e820bd1c6890f93ea1a84ffd5730bbf0e3d8e153",
"script":"GATK2-bqsr",
"script_parameters":{
"input":{
- "output_of":"GATK2-realign"
+ "output_of":"GATK2-realign"
},
"gatk_bundle":{
- "optional":false
+ "value":"d237a90bae3870b3b033aea1e99de4a9+10820+K@qr1hi",
+ "required":true
},
"picard_zip":{
- "optional":false
+ "value":"687f74675c6a0e925dec619cc2bec25f+77",
+ "required":true
},
"gatk_tbz":{
- "optional":false
+ "value":"7e0a277d6d2353678a11f56bab3b13f2+87",
+ "required":true
}
- }
+ },
+ "output_is_persistent":false
},
"GATK2-merge-call":{
- "script_version":"82a471c92036198aaf02ca0467ea48d49dbe822d",
+ "repository":"arvados",
+ "script_version":"e820bd1c6890f93ea1a84ffd5730bbf0e3d8e153",
"script":"GATK2-merge-call",
"script_parameters":{
"input":{
- "output_of":"GATK2-bqsr"
+ "output_of":"GATK2-bqsr"
},
"gatk_bundle":{
- "optional":false
+ "value":"d237a90bae3870b3b033aea1e99de4a9+10820+K@qr1hi",
+ "required":true
},
"picard_zip":{
- "optional":false
+ "value":"687f74675c6a0e925dec619cc2bec25f+77",
+ "required":true
},
"gatk_tbz":{
- "optional":false
+ "value":"7e0a277d6d2353678a11f56bab3b13f2+87",
+ "required":true
},
"regions":{
- "optional":true
+ "value":"13b53dbe1ec032dfc495fd974aa5dd4a+87/S02972011_Covered_sort_merged.bed"
},
"region_padding":{
- "optional":true
+ "value":10
},
"GATK2_UnifiedGenotyper_args":{
- "default":["-stand_call_conf","30.0","-stand_emit_conf","30.0","-dcov","200"]
+ "default":[
+ "-stand_call_conf",
+ "30.0",
+ "-stand_emit_conf",
+ "30.0",
+ "-dcov",
+ "200"
+ ]
}
- }
+ },
+ "output_is_persistent":true
}
}
}
navsection: api
navmenu: Schema
title: {resource}
-navorder: {navorder}
---
h1. {resource}
h2. Methods
-See "REST methods for working with Arvados resources":/api/methods.html
+ See "REST methods for working with Arvados resources":{{{{site.baseurl}}}}/api/methods.html
API endpoint base: @https://{{{{ site.arvados_api_host }}}}/arvados/v1/{res_api_endpoint}@
<p>manuals, guides, and references</p>
</div>
<div class="col-sm-6">
- <img src="images/dax.png" style="max-height: 10em"></img>
+ <img src="images/dax-reading-book.png" style="max-height: 10em" alt="Dax reading a book" />
</div>
</div>
</div>
</div>
<div class="col-sm-7" style="border-left: solid; border-width: 1px">
<p>
- <a href="{{ site.baseurl }}/user/">User Guide</a> — How to manage data and do analysis with Arvados.
+ <a href="{{ site.baseurl }}/user/index.html">User Guide</a> — How to manage data and do analysis with Arvados.
</p>
<p>
- <a href="{{ site.baseurl }}/sdk/">SDK Reference</a> — Details about the accessing Arvados from various programming languages.
+ <a href="{{ site.baseurl }}/sdk/index.html">SDK Reference</a> — Details about the accessing Arvados from various programming languages.
</p>
<p>
- <a href="{{ site.baseurl }}/api/">API Reference</a> — Details about the the Arvados REST API.
+ <a href="{{ site.baseurl }}/api/index.html">API Reference</a> — Details about the the Arvados REST API.
</p>
<p>
- <a href="{{ site.baseurl }}/admin/">Admin Guide</a> — How to administer an Arvados system.
+ <a href="{{ site.baseurl }}/admin/index.html">Admin Guide</a> — How to administer an Arvados system.
</p>
<p>
- <a href="{{ site.baseurl }}/install/">Install Guide</a> — How to install Arvados on a cloud platform.
- </p>
+ <a href="{{ site.baseurl }}/install/index.html">Install Guide</a> — How to install Arvados on a cloud platform.
+ </p>
</div>
</div>
</div>
...
-h1. Install client libraries
+
h3. Python
{% include 'notebox_end' %}
notextile. <pre><code>$ <span class="userinput">sudo gem install arvados arvados-cli</span></code></pre>
+
+h3. Perl
+
+{% include 'notebox_begin' %}
+The Perl client library includes the @Arvados.pm@ module and submodules.
+{% include 'notebox_end' %}
+
+<notextile>
+<pre><code>$ <span class="userinput">cd arvados/sdk/perl</span>
+$ <span class="userinput">perl Makefile.PL</span>
+$ <span class="userinput">sudo make install</span>
+</code></pre>
+</notextile>
+
...
-h1. Create standard objects
+
h3. "All users" group
read -rd $'\000' newlink <<EOF; arv link create --link "$newlink"
{
- "tail_kind":"arvados#group",
"tail_uuid":"$all_users_group_uuid",
- "head_kind":"arvados#repository",
"head_uuid":"$repo_uuid",
"link_class":"permission",
"name":"can_read"
1. Set up a cluster, or use Amazon
1. Create and mount Keep volumes
+1. [Install the Single Sign On (SSO) server](install-sso.html)
1. [Install the Arvados REST API server](install-api-server.html)
1. [Install the Arvados workbench application](install-workbench-app.html)
1. [Install the Crunch dispatcher](install-crunch-dispatch.html)
+++ /dev/null
----
-layout: default
-navsection: installguide
-title: Install the API server
-navorder: 1
-...
-
-{% include 'alert_stub' %}
-
-# API server setup
-
-## Prerequisites
-
-1. A GNU/linux (virtual) machine
-2. A domain name for your api server
-
-## Download the source tree
-
-Please follow the instructions on the [Download page](https://arvados.org/projects/arvados/wiki/Download) in the wiki.
-
-## Configure the API server
-
-First install the gems:
-
- cd arvados/services/api
- bundle install
-
-Next, configure the database:
-
- cp config/database.yml.sample config/database.yml
-
-Edit database.yml to your liking and make sure the database and db user exist.
-Then set up the database:
-
- RAILS_ENV=production rake db:setup
-
-Then set up omniauth:
-
- cp config/initializers/omniauth.rb.example config/initializers/omniauth.rb
-
-Edit config/initializers/omniauth.rb. Choose an *APP_SECRET* and *APP_ID*. Also set
-*CUSTOM_PROVIDER_URL*.
-
-Make sure your Omniauth provider knows about your *APP_ID* and *APP_SECRET*
-combination.
-
-You also need to update config/initializers/secret_token.rb. Generate a new secret with
-
- rake secret
-
-and put it in config/initializers/secret_token.rb:
-
- Server::Application.config.secret_token = 'your-new-secret-here'
-
-Finally, edit the main configuration:
-
- cp config/environments/production.rb.example config/environments/production.rb
-
-First, you want to make sure that
-
- config.uuid_prefix
-
-is set to a unique 5-digit hex string. You can replace the 'cfi-aws-0' string
-with a string of your choice to make that happen.
-
-The *config.uuid_prefix* string is a unique identifier for your API server. It
-also serves as the first part of the hostname for your API server, for instance
-
- {{ site.arvados_api_host }}
-
-You should use your own domain instead of arvadosapi.com
-
-Second, unless you are running on AWS, you will want to change the definition of
-
- config.compute_node_nameservers
-
-If you know your nameservers and they are fixed, you can hardcode them, and
-make sure to remove the code that tries to look them up from the AWS metadata:
-
- config.compute_node_nameservers = ['1.2.3.4','2.3.4.5','3.4.5.6']
- #require 'net/http'
- #config.compute_node_nameservers = ['local', 'public'].collect do |iface|
- # Net::HTTP.get(URI("http://169.254.169.254/latest/meta-data/#{iface}-ipv4")).match(/^[\d\.]+$/)[0]
- #end << '172.16.0.23'
-
-## Apache/Passenger
-
-Set up Apache and Passenger. Point them to the services/api directory in the source tree.
-
-To enable streaming so users can monitor crunch jobs in real time, add
-to your Apache configuration:
-
- PassengerBufferResponse off
-
-## Add an admin user
-
-Point browser to the API endpoint. Log in with a google account.
-
-In the rails console:
-
- Thread.current[:user] = User.find(1)
- Thread.current[:user].is_admin = true
- User.find(1).update_attributes is_admin: true, is_active: true
- User.find(1).is_admin
-
-This should be
-
- => true
-
-## Create a token
-
-In rails console
-
- a = ApiClient.new(owner_uuid:'0'); a.save!
- x = ApiClientAuthorization.new(api_client_id:a.id, user_id:1); x.save; x.api_token
-
--- /dev/null
+---
+layout: default
+navsection: installguide
+title: Install the API server
+...
+
+h2. Prerequisites:
+
+# A GNU/Linux (virtual) machine
+# A domain name for your api server
+# Ruby >= 2.0.0
+# Bundler: @gem install bundler@
+# Curl libraries: @sudo apt-get install libcurl3 libcurl3-gnutls libcurl4-openssl-dev@
+
+h2. Download the source tree
+
+<notextile>
+<pre><code>~$ <span class="userinput">git clone https://github.com/curoverse/arvados.git</span>
+</code></pre></notextile>
+
+See also: "Downloading the source code:https://arvados.org/projects/arvados/wiki/Download on the Arvados wiki.
+
+h2. Install gem dependencies
+
+<notextile>
+<pre><code>~$ <span class="userinput">cd arvados/services/api</span>
+~/arvados/services/api$ <span class="userinput">bundle install</span>
+</code></pre></notextile>
+
+h2. Configure the API server
+
+Edit the main configuration:
+
+<notextile>
+<pre><code>~/arvados/services/api$ <span class="userinput">cp -i config/application.yml.example config/application.yml</span>
+</code></pre></notextile>
+
+Choose a unique 5-character alphanumeric string to use as your @uuid_prefix@. An example is given that generates a 5-character string based on a hash of your hostname. The @uuid_prefix@ is a unique identifier for your API server. It also serves as the first part of the hostname for your API server.
+
+For a development site, use your own domain instead of arvadosapi.com.
+
+Make sure a clone of the arvados repository exists in @git_repositories_dir@:
+
+<notextile>
+<pre><code>~/arvados/services/api$ <span class="userinput">sudo mkdir -p /var/cache/git</span>
+~/arvados/services/api$ <span class="userinput">sudo git clone --bare ../../.git /var/cache/git/arvados.git</span>
+</code></pre></notextile>
+
+Generate a new secret token for signing cookies:
+
+<notextile>
+<pre><code>~/arvados/services/api$ <span class="userinput">rake secret
+zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz
+</code></pre></notextile>
+
+Put it in @config/application.yml@ in the production or common section:
+
+<notextile>
+<pre><code><span class="userinput"> secret_token: zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz</span>
+</code></pre>
+</notextile>
+
+Consult @application.default.yml@ for a full list of configuration options. Always put your local configuration in @application.yml@ instead of editing @application.default.yml@.
+
+Configure the database:
+
+<notextile>
+<pre><code>~/arvados/services/api$ <span class="userinput">cp -i config/database.yml.sample config/database.yml</span>
+</code></pre></notextile>
+
+By default, the development database will use the sqlite3 driver, so no configuration is necessary. If you wish to use mysql or postgres, edit @config/database.yml@ to your liking and make sure the database and db user exist. Then initialize the database:
+
+<notextile>
+<pre><code>~/arvados/services/api$ <span class="userinput">RAILS_ENV=development bundle exec rake db:setup</span>
+</code></pre></notextile>
+
+Set up omniauth:
+
+<notextile>
+<pre><code>~/arvados/services/api$ <span class="userinput">cp -i config/initializers/omniauth.rb.example config/initializers/omniauth.rb
+</code></pre></notextile>
+
+Edit @config/initializers/omniauth.rb@. Set @APP_SECRET@ to the value of @app_secret@ from "installing the single sign on server":install-sso.html .
+
+You can now run the development server:
+
+<notextile>
+<pre><code>~/arvados/services/api$ <span class="userinput">rails server
+</code></pre></notextile>
+
+h3. Apache/Passenger (optional)
+
+You can use "Passenger":https://www.phusionpassenger.com/ for deployment. Point it to the services/api directory in the source tree.
+
+To enable streaming so users can monitor crunch jobs in real time, add to your Passenger configuration in Apache:
+
+<notextile>
+<pre><code><span class="userinput">PassengerBufferResponse off</span>
+</code></pre>
+</notextile>
+
+h2. Add an admin user
+
+Point browser to the API endpoint. Log in with a google account.
+
+In the rails console:
+
+<notextile>
+<pre><code>~/arvados/services/api$ <span class="userinput">rails console</span>
+irb(main):001:0> <span class="userinput">Thread.current[:user] = User.find(1)</span>
+irb(main):002:0> <span class="userinput">Thread.current[:user].is_admin = true</span>
+irb(main):003:0> <span class="userinput">User.find(1).update_attributes is_admin: true, is_active: true</span>
+irb(main):004:0> <span class="userinput">User.find(1).is_admin</span>
+=> true
+</code></pre></notextile>
+
+h2. Create an API token
+
+In rails console:
+
+<notextile>
+<pre><code>~/arvados/services/api$ <span class="userinput">rails console</span>
+irb(main):001:0> <span class="userinput">a = ApiClient.new(owner_uuid:'0')</span>
+irb(main):002:0> <span class="userinput">a.save!</span>
+irb(main):003:0> <span class="userinput">x = ApiClientAuthorization.new(api_client_id:a.id, user_id:1)</span>
+irb(main):004:0> <span class="userinput">x.save</span>
+irb(main):005:0> <span class="userinput">x.api_token</span>
+</code></pre></notextile>
...
-h1. Crunch setup
+
The dispatcher normally runs on the same host/VM as the API server.
--- /dev/null
+---
+layout: default
+navsection: installguide
+title: Install Single Sign On (SSO) server
+...
+
+<notextile>
+<pre><code>~$ <span class="userinput">git clone https://github.com/curoverse/sso-devise-omniauth-provider.git</span>
+~$ <span class="userinput">cd sso-devise-omniauth-provider</span>
+~/sso-devise-omniauth-provider$ <span class="userinput">bundle install</span>
+~/sso-devise-omniauth-provider$ <span class="userinput">rake db:create</span>
+~/sso-devise-omniauth-provider$ <span class="userinput">rake db:migrate</span>
+~/sso-devise-omniauth-provider$ <span class="userinput">rake secret</span>
+xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx
+~/sso-devise-omniauth-provider$ <span class="userinput">rails console</span>
+irb(main):001:0> <span class="userinput">c = Client.new</span>
+irb(main):002:0> <span class="userinput">c.name = "joshid"</span>
+irb(main):003:0> <span class="userinput">c.app_id = "arvados-server"</span>
+irb(main):004:0> <span class="userinput">c.app_secret = "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx"</span>
+irb(main):005:0> <span class="userinput">c.save!</span>
+irb(main):006:0> <span class="userinput">quit</span>
+~/sso-devise-omniauth-provider$ <span class="userinput">rails server --port=3002</span>
+</code></pre>
+</notextile>
+++ /dev/null
----
-layout: default
-navsection: installguide
-title: Install the Arvados workbench application
-navorder: 2
-...
-
-{% include 'alert_stub' %}
-
-# Workbench setup
-
-## Prerequisites
-
-1. A GNU/linux (virtual) machine (can be shared with the API server)
-2. A hostname for your workbench application
-
-## Download the source tree
-
-Please follow the instructions on the [Download page](https://arvados.org/projects/arvados/wiki/Download) in the wiki.
-
-The workbench application is in `apps/workbench`.
-
-## Configure the Workbench application
-
-You need to update config/initializers/secret_token.rb. Generate a new secret with
-
- rake secret
-
-and put it in `config/initializers/secret_token.rb`
-
- Server::Application.config.secret_token = 'your-new-secret-here'
-
-Adjust the following fields in your `environments/production.rb` file.
-
-* `config.site_name` can be the URL of your workbench install.
-* `config.arvados_login_base` and `config.arvados_v1_base` should point to
-your API server. Use the example values as a guide.
-* If you choose not to use https, make sure to also set
-`config.force_ssl = false` in the API server's `production.rb` file.
-
-Copy `config/piwik.yml.example` to `config/piwik.yml` and edit to suit.
-
-## Apache/Passenger
-
-Set up Apache and Passenger. Point them to the apps/workbench directory in the source tree.
-
-## "Trusted client" setting
-
-Log in to Workbench once (this ensures that the Arvados API server has
-a record of the Workbench client).
-
-In the API server project root, start the rails console.
-
- RAILS_ENV=production bundle exec rails c
-
-Locate the ApiClient record for your Workbench installation.
-
- ApiClient.where('url_prefix like ?', '%workbench%')
-
-Set the `is_trusted` flag for the appropriate client record.
-
- ApiClient.find(1234).update_attributes is_trusted: true
-
--- /dev/null
+---
+layout: default
+navsection: installguide
+title: Install the Arvados Workbench application
+...
+
+h2. Prerequisites
+
+# A GNU/linux (virtual) machine (can be shared with the API server)
+# A hostname for your Workbench application
+
+h2. Download the source tree
+
+Please follow the instructions on the "Download page":https://arvados.org/projects/arvados/wiki/Download in the wiki.
+
+The Workbench application is in @arvados/apps/workbench@.
+
+h2. Configure the Workbench application
+
+This application needs a secret token. Generate a new secret:
+
+<notextile>
+<pre><code>~/arvados/apps/workbench$ <span class="userinput">rake secret</span>
+aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa
+</code></pre>
+</notextile>
+
+Copy @config/application.yml.example@ to @config/application.yml@ and edit it appropriately for your environment.
+
+* Set @secret_token@ to the string you generated with @rake secret@.
+* Point @arvados_login_base@ and @arvados_v1_base@ at your "API server":install-api-server.html
+* @site_name@ can be any string to identify this Workbench.
+* Assuming that the SSL certificate you use for development isn't signed by a CA, make sure @arvados_insecure_https@ is @true@.
+
+Copy @config/piwik.yml.example@ to @config/piwik.yml@ and edit to suit.
+
+h3. Apache/Passenger (optional)
+
+Set up Apache and Passenger. Point them to the apps/workbench directory in the source tree.
+
+h2. Trusted client setting
+
+Log in to Workbench once (this ensures that the Arvados API server has a record of the Workbench client).
+
+In the API server project root, start the rails console. Locate the ApiClient record for your Workbench installation, then set the `is_trusted` flag for the appropriate client record:
+
+<notextile><pre><code>~/arvados/services/api$ <span class="userinput">RAILS_ENV=development bundle exec rails console</span>
+irb(main):001:0> <span class="userinput">ApiClient.where('url_prefix like ?', '%workbench%')</span>
+=> {:id => 1234}
+irb(main):002:0> <span class="userinput">ApiClient.find(1234).update_attributes is_trusted: true</span>
+</code></pre>
+</notextile>
--- /dev/null
+---
+layout: default
+navsection: sdk
+navmenu: CLI
+title: "Command line SDK"
+
+...
+
+The @arv@ CLI tool provides a generic set of wrappers so you can make API calls easily. It performs some validation before connecting to the API server: for example, it refuses to do an API call if a required parameter is missing.
+
+It also provides access to Keep storage services with the @arv keep@ subcommand.
+
+h3. Usage
+
+See the "command line interface":{{site.baseurl}}/user/reference/sdk-cli.html page in the user guide.
+
+h3. Installation
+
+If you are logged in to an Arvados VM, the @arv@ should be installed.
+
+To use @arv@ elsewhere, you can either install the @arvados-cli@ gem via RubyGems or build and install the package using the arvados source tree.
+
+h4. Prerequisites: Ruby >= 2.0.0 and curl libraries
+
+You can use "RVM":http://rvm.io/rvm/install to install and manage Ruby versions.
+
+<notextile>
+<pre>
+$ <code class="userinput">sudo apt-get install curl</code>
+$ <code class="userinput">sudo sh -c 'curl -sSL https://get.rvm.io | bash -s stable'</code>
+$ <code class="userinput">source /etc/profile.d/rvm.sh</code>
+</pre>
+</notextile>
+
+Install curl libraries with your system's package manager. For example, with Debian or Ubuntu:
+
+<notextile>
+<pre>
+$ <code class="userinput">sudo apt-get install libcurl3 libcurl3-gnutls libcurl4-openssl-dev</code>
+</pre>
+</notextile>
+
+h4. Option 1: install with RubyGems
+
+<notextile>
+<pre>
+$ <code class="userinput">sudo gem install arvados-cli</code>
+</pre>
+</notextile>
+
+h4. Option 2: build and install from source
+
+<notextile>
+<pre>
+$ <code class="userinput">git clone https://github.com/curoverse/arvados.git</code>
+$ <code class="userinput">cd arvados/sdk/cli</code>
+$ <code class="userinput">gem build arvados-cli.gemspec</code>
+$ <code class="userinput">sudo gem install arvados-cli-*.gem</code>
+</pre>
+</notextile>
---
layout: default
navsection: sdk
-title: "SDK Reference"
-
+title: "Arvados SDK Reference"
...
-h1. Arvados SDK Reference
-
This section documents how to access the Arvados API and Keep using various programming languages.
-* "Python SDK":python/sdk-python.html
+* "Python SDK":{{site.baseurl}}/sdk/python/sdk-python.html
+* "Perl SDK":{{site.baseurl}}/sdk/perl/index.html
+* "Ruby SDK":{{site.baseurl}}/sdk/ruby/index.html
+* "Command line SDK":{{site.baseurl}}/sdk/cli/index.html ("arv")
+
+SDKs not yet implemented:
+
+* Rails SDK: Workbench uses an ActiveRecord-like interface to Arvados. This hasn't yet been extracted from Workbench and packaged as a gem.
+* R and Java: We plan to support these, but they have not been implemented yet.
--- /dev/null
+---
+layout: default
+navsection: sdk
+navmenu: Perl
+title: "Perl SDK"
+
+...
+
+The Perl SDK provides a generic set of wrappers so you can make API calls easily.
+
+It should be treated as alpha/experimental. Currently, limitations include:
+* Verbose syntax.
+* No native Keep client.
+* No CPAN package.
+
+h3. Installation
+
+<notextile>
+<pre>
+$ <code class="userinput">sudo apt-get install libjson-perl libio-socket-ssl-perl libwww-perl</code>
+$ <code class="userinput">git clone https://github.com/curoverse/arvados.git</code>
+$ <code class="userinput">cd arvados/sdk/perl</code>
+$ <code class="userinput">perl Makefile.PL</code>
+$ <code class="userinput">sudo make install</code>
+</pre>
+</notextile>
+
+h4. Test installation
+
+If the SDK is installed, @perl -MArvados -e ''@ should produce no errors.
+
+If your @ARVADOS_API_HOST@ and @ARVADOS_API_TOKEN@ environment variables are set up correctly (see "api-tokens":{{site.baseurl}}/user/reference/api-tokens.html for details), the following test script should work:
+
+<notextile>
+<pre>$ <code class="userinput">perl <<'EOF'
+use Arvados;
+my $arv = Arvados->new('apiVersion' => 'v1');
+my $me = $arv->{'users'}->{'current'}->execute;
+print ("arvados.v1.users.current.full_name = '", $me->{'full_name'}, "'\n");
+EOF</code>
+arvados.v1.users.current.full_name = 'Your Name'
+</pre>
+</notextile>
+
+h3. Examples
+
+Set up an API client user agent:
+
+<notextile>
+<pre><code class="userinput">my $arv = Arvados->new('apiVersion' => 'v1');
+</code></pre>
+</notextile>
+
+Get the User object for the current user:
+
+<notextile>
+<pre><code class="userinput">my $current_user = $arv->{'users'}->{'current'}->execute;
+</code></pre>
+</notextile>
+
+Get the UUID of an object that was retrieved using the SDK:
+
+<notextile>
+<pre><code class="userinput">my $current_user_uuid = $current_user->{'uuid'}
+</code></pre>
+</notextile>
+
+Retrieve an object by ID:
+
+<notextile>
+<pre><code class="userinput">my $some_user = $arv->{'users'}->{'get'}->execute('uuid' => $current_user_uuid);
+</code></pre>
+</notextile>
+
+Create an object:
+
+<notextile>
+<pre><code class="userinput">my $test_link = $arv->{'links'}->{'create'}->execute('link' => { 'link_class' => 'test', 'name' => 'test' });
+</code></pre>
+</notextile>
+
+Update an object:
+
+<notextile>
+<pre><code class="userinput">my $test_link = $arv->{'links'}->{'update'}->execute(
+ 'uuid' => $test_link->{'uuid'},
+ 'link' => { 'properties' => { 'foo' => 'bar' } });
+</code></pre>
+</notextile>
+
+Get a list of objects:
+
+<notextile>
+<pre><code class="userinput">my $repos = $arv->{'repositories'}->{'list'}->execute;
+print ("UUID of first repo returned is ", $repos->{'items'}->[0], "\n");
+</code></pre>
+</notextile>
+
+The SDK retrieves the list of API methods from the server at run time. Therefore, the set of available methods is determined by the server version rather than the SDK version.
...
-h1. Crunch utility libraries
-
Several utility libraries are included with Arvados. They are intended to make it quicker and easier to write your own crunch scripts.
* "Python SDK extras":#pythonsdk
...
-h1. Python SDK
-
The Python SDK provides a generic set of wrappers so you can make API calls easily. It performs some validation before connecting to the API server: for example, it refuses to do an API call if a required parameter is missing.
The library also includes some conveniences for use in Crunch scripts; see "Crunch utility libraries":crunch-utility-libraries.html for details.
<notextile>
<pre>
-$ <code class="userinput">sudo apt-get install python-dev libattr1-dev libfuse-dev pkg-config</code>
+$ <code class="userinput">sudo apt-get install python-pip python-dev libattr1-dev libfuse-dev pkg-config</code>
$ <code class="userinput">sudo pip install arvados-python-client</code>
</pre>
</notextile>
+_If your version of @pip@ is 1.4 or newer, the @pip install@ command might give an error: "Could not find a version that satisfies the requirement arvados-python-client". If this happens, fix it by adding a @--pre@ flag:_
+
+<notextile>
+<pre>
+$ <code class="userinput">sudo pip install --pre arvados-python-client</code>
+</pre>
+</notextile>
+
h4. Option 2: build and install from source
<notextile>
--- /dev/null
+---
+layout: default
+navsection: sdk
+navmenu: Ruby
+title: "Ruby SDK"
+
+...
+
+The Ruby SDK provides a generic set of wrappers so you can make API calls easily.
+
+h3. Installation
+
+If you are logged in to an Arvados VM, the Ruby SDK should be installed.
+
+To use it elsewhere, you can either install the @arvados@ gem via RubyGems or build and install the package using the arvados source tree.
+
+h4. Prerequisites: Ruby >= 2.0.0
+
+You can use "RVM":http://rvm.io/rvm/install to install and manage Ruby versions.
+
+h4. Option 1: install with RubyGems
+
+<notextile>
+<pre>
+$ <code class="userinput">sudo gem install arvados</code>
+</pre>
+</notextile>
+
+h4. Option 2: build and install from source
+
+<notextile>
+<pre>
+$ <code class="userinput">git clone https://github.com/curoverse/arvados.git</code>
+$ <code class="userinput">cd arvados/sdk/cli</code>
+$ <code class="userinput">gem build arvados.gemspec</code>
+$ <code class="userinput">sudo gem install arvados-*.gem</code>
+</pre>
+</notextile>
+
+h4. Test installation
+
+If the SDK is installed, @ruby -r arvados -e 'puts "OK!"'@ should produce no errors.
+
+If your @ARVADOS_API_HOST@ and @ARVADOS_API_TOKEN@ environment variables are set up correctly (see "api-tokens":{{site.baseurl}}/user/reference/api-tokens.html for details), the following test script should work:
+
+<notextile>
+<pre>$ <code class="userinput">ruby -r arvados <<'EOF'
+arv = Arvados.new api_version: 'v1'
+my_full_name = arv.user.current[:full_name]
+puts "arvados.v1.users.current.full_name = '#{my_full_name}'"
+EOF</code>
+arvados.v1.users.current.full_name = 'Your Name'
+</pre>
+</notextile>
+
+h3. Examples
+
+Import the module (we skipped this step above by using "ruby -r arvados"):
+
+<notextile>
+<pre><code class="userinput">require 'arvados'
+</code></pre>
+</notextile>
+
+Set up an API client user agent:
+
+<notextile>
+<pre><code class="userinput">arv = Arvados.new(apiVersion: 'v1')
+</code></pre>
+</notextile>
+
+Get the User object for the current user:
+
+<notextile>
+<pre><code class="userinput">current_user = arv.user.current
+</code></pre>
+</notextile>
+
+Get the UUID of an object that was retrieved using the SDK:
+
+<notextile>
+<pre><code class="userinput">current_user_uuid = current_user[:uuid]
+</code></pre>
+</notextile>
+
+Retrieve an object by ID:
+
+<notextile>
+<pre><code class="userinput">some_user = arv.user.get(uuid: current_user_uuid)
+</code></pre>
+</notextile>
+
+Create an object:
+
+<notextile>
+<pre><code class="userinput">new_link = arv.link.create(link: {link_class: 'test', name: 'test'})
+</code></pre>
+</notextile>
+
+Update an object:
+
+<notextile>
+<pre><code class="userinput">updated_link = arv.link.update(uuid: new_link[:uuid],
+ link: {properties: {foo: 'bar'}})
+</code></pre>
+</notextile>
+
+Delete an object:
+
+<notextile>
+<pre><code class="userinput">arv.link.delete(uuid: new_link[:uuid])
+</code></pre>
+</notextile>
+
+Get a list of objects:
+
+<notextile>
+<pre><code class="userinput">repos = arv.repository.list
+first_repo = repos[:items][0]
+puts "UUID of first repo returned is #{first_repo[:uuid]}"</code>
+UUID of first repo returned is qr1hi-s0uqq-b1bnybpx3u5temz
+</pre>
+</notextile>
+
+The SDK retrieves the list of API methods from the server at run time. Therefore, the set of available methods is determined by the server version rather than the SDK version.
--- /dev/null
+---
+layout: default
+navsection: userguide
+title: "Apache License"
+...
+
+<div id="content" class="grid_16"><div class="section-content"></br>Version 2.0, January 2004<br></br>
+<a href="http://www.apache.org/licenses/">http://www.apache.org/licenses/</a> </p>
+<p>TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION</p>
+<p><strong><a name="definitions">1. Definitions</a></strong>.</p>
+<p>"License" shall mean the terms and conditions for use, reproduction, and
+distribution as defined by Sections 1 through 9 of this document.</p>
+<p>"Licensor" shall mean the copyright owner or entity authorized by the
+copyright owner that is granting the License.</p>
+<p>"Legal Entity" shall mean the union of the acting entity and all other
+entities that control, are controlled by, or are under common control with
+that entity. For the purposes of this definition, "control" means (i) the
+power, direct or indirect, to cause the direction or management of such
+entity, whether by contract or otherwise, or (ii) ownership of fifty
+percent (50%) or more of the outstanding shares, or (iii) beneficial
+ownership of such entity.</p>
+<p>"You" (or "Your") shall mean an individual or Legal Entity exercising
+permissions granted by this License.</p>
+<p>"Source" form shall mean the preferred form for making modifications,
+including but not limited to software source code, documentation source,
+and configuration files.</p>
+<p>"Object" form shall mean any form resulting from mechanical transformation
+or translation of a Source form, including but not limited to compiled
+object code, generated documentation, and conversions to other media types.</p>
+<p>"Work" shall mean the work of authorship, whether in Source or Object form,
+made available under the License, as indicated by a copyright notice that
+is included in or attached to the work (an example is provided in the
+Appendix below).</p>
+<p>"Derivative Works" shall mean any work, whether in Source or Object form,
+that is based on (or derived from) the Work and for which the editorial
+revisions, annotations, elaborations, or other modifications represent, as
+a whole, an original work of authorship. For the purposes of this License,
+Derivative Works shall not include works that remain separable from, or
+merely link (or bind by name) to the interfaces of, the Work and Derivative
+Works thereof.</p>
+<p>"Contribution" shall mean any work of authorship, including the original
+version of the Work and any modifications or additions to that Work or
+Derivative Works thereof, that is intentionally submitted to Licensor for
+inclusion in the Work by the copyright owner or by an individual or Legal
+Entity authorized to submit on behalf of the copyright owner. For the
+purposes of this definition, "submitted" means any form of electronic,
+verbal, or written communication sent to the Licensor or its
+representatives, including but not limited to communication on electronic
+mailing lists, source code control systems, and issue tracking systems that
+are managed by, or on behalf of, the Licensor for the purpose of discussing
+and improving the Work, but excluding communication that is conspicuously
+marked or otherwise designated in writing by the copyright owner as "Not a
+Contribution."</p>
+<p>"Contributor" shall mean Licensor and any individual or Legal Entity on
+behalf of whom a Contribution has been received by Licensor and
+subsequently incorporated within the Work.</p>
+<p><strong><a name="copyright">2. Grant of Copyright License</a></strong>. Subject to the
+terms and conditions of this License, each Contributor hereby grants to You
+a perpetual, worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+copyright license to reproduce, prepare Derivative Works of, publicly
+display, publicly perform, sublicense, and distribute the Work and such
+Derivative Works in Source or Object form.</p>
+<p><strong><a name="patent">3. Grant of Patent License</a></strong>. Subject to the terms
+and conditions of this License, each Contributor hereby grants to You a
+perpetual, worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+(except as stated in this section) patent license to make, have made, use,
+offer to sell, sell, import, and otherwise transfer the Work, where such
+license applies only to those patent claims licensable by such Contributor
+that are necessarily infringed by their Contribution(s) alone or by
+combination of their Contribution(s) with the Work to which such
+Contribution(s) was submitted. If You institute patent litigation against
+any entity (including a cross-claim or counterclaim in a lawsuit) alleging
+that the Work or a Contribution incorporated within the Work constitutes
+direct or contributory patent infringement, then any patent licenses
+granted to You under this License for that Work shall terminate as of the
+date such litigation is filed.</p>
+<p><strong><a name="redistribution">4. Redistribution</a></strong>. You may reproduce and
+distribute copies of the Work or Derivative Works thereof in any medium,
+with or without modifications, and in Source or Object form, provided that
+You meet the following conditions:</p>
+<ol style="list-style: lower-latin;">
+<li>You must give any other recipients of the Work or Derivative Works a
+copy of this License; and</li>
+
+<li>You must cause any modified files to carry prominent notices stating
+that You changed the files; and</li>
+
+<li>You must retain, in the Source form of any Derivative Works that You
+distribute, all copyright, patent, trademark, and attribution notices from
+the Source form of the Work, excluding those notices that do not pertain to
+any part of the Derivative Works; and</li>
+
+<li>If the Work includes a "NOTICE" text file as part of its distribution,
+then any Derivative Works that You distribute must include a readable copy
+of the attribution notices contained within such NOTICE file, excluding
+those notices that do not pertain to any part of the Derivative Works, in
+at least one of the following places: within a NOTICE text file distributed
+as part of the Derivative Works; within the Source form or documentation,
+if provided along with the Derivative Works; or, within a display generated
+by the Derivative Works, if and wherever such third-party notices normally
+appear. The contents of the NOTICE file are for informational purposes only
+and do not modify the License. You may add Your own attribution notices
+within Derivative Works that You distribute, alongside or as an addendum to
+the NOTICE text from the Work, provided that such additional attribution
+notices cannot be construed as modifying the License.
+<br/>
+<br/>
+You may add Your own copyright statement to Your modifications and may
+provide additional or different license terms and conditions for use,
+reproduction, or distribution of Your modifications, or for any such
+Derivative Works as a whole, provided Your use, reproduction, and
+distribution of the Work otherwise complies with the conditions stated in
+this License.
+</li>
+
+</ol>
+
+<p><strong><a name="contributions">5. Submission of Contributions</a></strong>. Unless You
+explicitly state otherwise, any Contribution intentionally submitted for
+inclusion in the Work by You to the Licensor shall be under the terms and
+conditions of this License, without any additional terms or conditions.
+Notwithstanding the above, nothing herein shall supersede or modify the
+terms of any separate license agreement you may have executed with Licensor
+regarding such Contributions.</p>
+<p><strong><a name="trademarks">6. Trademarks</a></strong>. This License does not grant
+permission to use the trade names, trademarks, service marks, or product
+names of the Licensor, except as required for reasonable and customary use
+in describing the origin of the Work and reproducing the content of the
+NOTICE file.</p>
+<p><strong><a name="no-warranty">7. Disclaimer of Warranty</a></strong>. Unless required by
+applicable law or agreed to in writing, Licensor provides the Work (and
+each Contributor provides its Contributions) on an "AS IS" BASIS, WITHOUT
+WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied, including,
+without limitation, any warranties or conditions of TITLE,
+NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A PARTICULAR PURPOSE. You
+are solely responsible for determining the appropriateness of using or
+redistributing the Work and assume any risks associated with Your exercise
+of permissions under this License.</p>
+<p><strong><a name="no-liability">8. Limitation of Liability</a></strong>. In no event and
+under no legal theory, whether in tort (including negligence), contract, or
+otherwise, unless required by applicable law (such as deliberate and
+grossly negligent acts) or agreed to in writing, shall any Contributor be
+liable to You for damages, including any direct, indirect, special,
+incidental, or consequential damages of any character arising as a result
+of this License or out of the use or inability to use the Work (including
+but not limited to damages for loss of goodwill, work stoppage, computer
+failure or malfunction, or any and all other commercial damages or losses),
+even if such Contributor has been advised of the possibility of such
+damages.</p>
+<p><strong><a name="additional">9. Accepting Warranty or Additional Liability</a></strong>.
+While redistributing the Work or Derivative Works thereof, You may choose
+to offer, and charge a fee for, acceptance of support, warranty, indemnity,
+or other liability obligations and/or rights consistent with this License.
+However, in accepting such obligations, You may act only on Your own behalf
+and on Your sole responsibility, not on behalf of any other Contributor,
+and only if You agree to indemnify, defend, and hold each Contributor
+harmless for any liability incurred by, or claims asserted against, such
+Contributor by reason of your accepting any such warranty or additional
+liability.</p>
+<p>END OF TERMS AND CONDITIONS</p>
+<h1 id="apply">APPENDIX: How to apply the Apache License to your work</h1>
+<p>To apply the Apache License to your work, attach the following boilerplate
+notice, with the fields enclosed by brackets "[]" replaced with your own
+identifying information. (Don't include the brackets!) The text should be
+enclosed in the appropriate comment syntax for the file format. We also
+recommend that a file or class name and description of purpose be included
+on the same "printed page" as the copyright notice for easier
+identification within third-party archives.</p>
+<div class="codehilite"><pre>Copyright [yyyy] [name of copyright owner]
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+</pre></div></div></div>
--- /dev/null
+---
+layout: default
+navsection: userguide
+title: "GNU Affero General Public License"
+...
+
+<p style="text-align: center;">Version 3, 19 November 2007</p>
+
+<p>Copyright © 2007 Free Software Foundation,
+Inc. <<a href="http://www.fsf.org/">http://fsf.org/</a>>
+ <br />
+ Everyone is permitted to copy and distribute verbatim copies
+ of this license document, but changing it is not allowed.</p>
+
+<h3><a name="preamble"></a>Preamble</h3>
+
+<p>The GNU Affero General Public License is a free, copyleft license
+for software and other kinds of works, specifically designed to ensure
+cooperation with the community in the case of network server software.</p>
+
+<p>The licenses for most software and other practical works are
+designed to take away your freedom to share and change the works. By
+contrast, our General Public Licenses are intended to guarantee your
+freedom to share and change all versions of a program--to make sure it
+remains free software for all its users.</p>
+
+<p>When we speak of free software, we are referring to freedom, not
+price. Our General Public Licenses are designed to make sure that you
+have the freedom to distribute copies of free software (and charge for
+them if you wish), that you receive source code or can get it if you
+want it, that you can change the software or use pieces of it in new
+free programs, and that you know you can do these things.</p>
+
+<p>Developers that use our General Public Licenses protect your rights
+with two steps: (1) assert copyright on the software, and (2) offer
+you this License which gives you legal permission to copy, distribute
+and/or modify the software.</p>
+
+<p>A secondary benefit of defending all users' freedom is that
+improvements made in alternate versions of the program, if they
+receive widespread use, become available for other developers to
+incorporate. Many developers of free software are heartened and
+encouraged by the resulting cooperation. However, in the case of
+software used on network servers, this result may fail to come about.
+The GNU General Public License permits making a modified version and
+letting the public access it on a server without ever releasing its
+source code to the public.</p>
+
+<p>The GNU Affero General Public License is designed specifically to
+ensure that, in such cases, the modified source code becomes available
+to the community. It requires the operator of a network server to
+provide the source code of the modified version running there to the
+users of that server. Therefore, public use of a modified version, on
+a publicly accessible server, gives the public access to the source
+code of the modified version.</p>
+
+<p>An older license, called the Affero General Public License and
+published by Affero, was designed to accomplish similar goals. This is
+a different license, not a version of the Affero GPL, but Affero has
+released a new version of the Affero GPL which permits relicensing under
+this license.</p>
+
+<p>The precise terms and conditions for copying, distribution and
+modification follow.</p>
+
+<h3><a name="terms"></a>TERMS AND CONDITIONS</h3>
+
+<h4><a name="section0"></a>0. Definitions.</h4>
+
+<p>"This License" refers to version 3 of the GNU Affero General Public
+License.</p>
+
+<p>"Copyright" also means copyright-like laws that apply to other kinds
+of works, such as semiconductor masks.</p>
+
+<p>"The Program" refers to any copyrightable work licensed under this
+License. Each licensee is addressed as "you". "Licensees" and
+"recipients" may be individuals or organizations.</p>
+
+<p>To "modify" a work means to copy from or adapt all or part of the work
+in a fashion requiring copyright permission, other than the making of an
+exact copy. The resulting work is called a "modified version" of the
+earlier work or a work "based on" the earlier work.</p>
+
+<p>A "covered work" means either the unmodified Program or a work based
+on the Program.</p>
+
+<p>To "propagate" a work means to do anything with it that, without
+permission, would make you directly or secondarily liable for
+infringement under applicable copyright law, except executing it on a
+computer or modifying a private copy. Propagation includes copying,
+distribution (with or without modification), making available to the
+public, and in some countries other activities as well.</p>
+
+<p>To "convey" a work means any kind of propagation that enables other
+parties to make or receive copies. Mere interaction with a user through
+a computer network, with no transfer of a copy, is not conveying.</p>
+
+<p>An interactive user interface displays "Appropriate Legal Notices"
+to the extent that it includes a convenient and prominently visible
+feature that (1) displays an appropriate copyright notice, and (2)
+tells the user that there is no warranty for the work (except to the
+extent that warranties are provided), that licensees may convey the
+work under this License, and how to view a copy of this License. If
+the interface presents a list of user commands or options, such as a
+menu, a prominent item in the list meets this criterion.</p>
+
+<h4><a name="section1"></a>1. Source Code.</h4>
+
+<p>The "source code" for a work means the preferred form of the work
+for making modifications to it. "Object code" means any non-source
+form of a work.</p>
+
+<p>A "Standard Interface" means an interface that either is an official
+standard defined by a recognized standards body, or, in the case of
+interfaces specified for a particular programming language, one that
+is widely used among developers working in that language.</p>
+
+<p>The "System Libraries" of an executable work include anything, other
+than the work as a whole, that (a) is included in the normal form of
+packaging a Major Component, but which is not part of that Major
+Component, and (b) serves only to enable use of the work with that
+Major Component, or to implement a Standard Interface for which an
+implementation is available to the public in source code form. A
+"Major Component", in this context, means a major essential component
+(kernel, window system, and so on) of the specific operating system
+(if any) on which the executable work runs, or a compiler used to
+produce the work, or an object code interpreter used to run it.</p>
+
+<p>The "Corresponding Source" for a work in object code form means all
+the source code needed to generate, install, and (for an executable
+work) run the object code and to modify the work, including scripts to
+control those activities. However, it does not include the work's
+System Libraries, or general-purpose tools or generally available free
+programs which are used unmodified in performing those activities but
+which are not part of the work. For example, Corresponding Source
+includes interface definition files associated with source files for
+the work, and the source code for shared libraries and dynamically
+linked subprograms that the work is specifically designed to require,
+such as by intimate data communication or control flow between those
+subprograms and other parts of the work.</p>
+
+<p>The Corresponding Source need not include anything that users
+can regenerate automatically from other parts of the Corresponding
+Source.</p>
+
+<p>The Corresponding Source for a work in source code form is that
+same work.</p>
+
+<h4><a name="section2"></a>2. Basic Permissions.</h4>
+
+<p>All rights granted under this License are granted for the term of
+copyright on the Program, and are irrevocable provided the stated
+conditions are met. This License explicitly affirms your unlimited
+permission to run the unmodified Program. The output from running a
+covered work is covered by this License only if the output, given its
+content, constitutes a covered work. This License acknowledges your
+rights of fair use or other equivalent, as provided by copyright law.</p>
+
+<p>You may make, run and propagate covered works that you do not
+convey, without conditions so long as your license otherwise remains
+in force. You may convey covered works to others for the sole purpose
+of having them make modifications exclusively for you, or provide you
+with facilities for running those works, provided that you comply with
+the terms of this License in conveying all material for which you do
+not control copyright. Those thus making or running the covered works
+for you must do so exclusively on your behalf, under your direction
+and control, on terms that prohibit them from making any copies of
+your copyrighted material outside their relationship with you.</p>
+
+<p>Conveying under any other circumstances is permitted solely under
+the conditions stated below. Sublicensing is not allowed; section 10
+makes it unnecessary.</p>
+
+<h4><a name="section3"></a>3. Protecting Users' Legal Rights From Anti-Circumvention Law.</h4>
+
+<p>No covered work shall be deemed part of an effective technological
+measure under any applicable law fulfilling obligations under article
+11 of the WIPO copyright treaty adopted on 20 December 1996, or
+similar laws prohibiting or restricting circumvention of such
+measures.</p>
+
+<p>When you convey a covered work, you waive any legal power to forbid
+circumvention of technological measures to the extent such circumvention
+is effected by exercising rights under this License with respect to
+the covered work, and you disclaim any intention to limit operation or
+modification of the work as a means of enforcing, against the work's
+users, your or third parties' legal rights to forbid circumvention of
+technological measures.</p>
+
+<h4><a name="section4"></a>4. Conveying Verbatim Copies.</h4>
+
+<p>You may convey verbatim copies of the Program's source code as you
+receive it, in any medium, provided that you conspicuously and
+appropriately publish on each copy an appropriate copyright notice;
+keep intact all notices stating that this License and any
+non-permissive terms added in accord with section 7 apply to the code;
+keep intact all notices of the absence of any warranty; and give all
+recipients a copy of this License along with the Program.</p>
+
+<p>You may charge any price or no price for each copy that you convey,
+and you may offer support or warranty protection for a fee.</p>
+
+<h4><a name="section5"></a>5. Conveying Modified Source Versions.</h4>
+
+<p>You may convey a work based on the Program, or the modifications to
+produce it from the Program, in the form of source code under the
+terms of section 4, provided that you also meet all of these conditions:</p>
+
+<ul>
+
+<li>a) The work must carry prominent notices stating that you modified
+ it, and giving a relevant date.</li>
+
+<li>b) The work must carry prominent notices stating that it is
+ released under this License and any conditions added under section
+ 7. This requirement modifies the requirement in section 4 to
+ "keep intact all notices".</li>
+
+<li>c) You must license the entire work, as a whole, under this
+ License to anyone who comes into possession of a copy. This
+ License will therefore apply, along with any applicable section 7
+ additional terms, to the whole of the work, and all its parts,
+ regardless of how they are packaged. This License gives no
+ permission to license the work in any other way, but it does not
+ invalidate such permission if you have separately received it.</li>
+
+<li>d) If the work has interactive user interfaces, each must display
+ Appropriate Legal Notices; however, if the Program has interactive
+ interfaces that do not display Appropriate Legal Notices, your
+ work need not make them do so.</li>
+
+</ul>
+
+<p>A compilation of a covered work with other separate and independent
+works, which are not by their nature extensions of the covered work,
+and which are not combined with it such as to form a larger program,
+in or on a volume of a storage or distribution medium, is called an
+"aggregate" if the compilation and its resulting copyright are not
+used to limit the access or legal rights of the compilation's users
+beyond what the individual works permit. Inclusion of a covered work
+in an aggregate does not cause this License to apply to the other
+parts of the aggregate.</p>
+
+<h4><a name="section6"></a>6. Conveying Non-Source Forms.</h4>
+
+<p>You may convey a covered work in object code form under the terms
+of sections 4 and 5, provided that you also convey the
+machine-readable Corresponding Source under the terms of this License,
+in one of these ways:</p>
+
+<ul>
+
+<li>a) Convey the object code in, or embodied in, a physical product
+ (including a physical distribution medium), accompanied by the
+ Corresponding Source fixed on a durable physical medium
+ customarily used for software interchange.</li>
+
+<li>b) Convey the object code in, or embodied in, a physical product
+ (including a physical distribution medium), accompanied by a
+ written offer, valid for at least three years and valid for as
+ long as you offer spare parts or customer support for that product
+ model, to give anyone who possesses the object code either (1) a
+ copy of the Corresponding Source for all the software in the
+ product that is covered by this License, on a durable physical
+ medium customarily used for software interchange, for a price no
+ more than your reasonable cost of physically performing this
+ conveying of source, or (2) access to copy the
+ Corresponding Source from a network server at no charge.</li>
+
+<li>c) Convey individual copies of the object code with a copy of the
+ written offer to provide the Corresponding Source. This
+ alternative is allowed only occasionally and noncommercially, and
+ only if you received the object code with such an offer, in accord
+ with subsection 6b.</li>
+
+<li>d) Convey the object code by offering access from a designated
+ place (gratis or for a charge), and offer equivalent access to the
+ Corresponding Source in the same way through the same place at no
+ further charge. You need not require recipients to copy the
+ Corresponding Source along with the object code. If the place to
+ copy the object code is a network server, the Corresponding Source
+ may be on a different server (operated by you or a third party)
+ that supports equivalent copying facilities, provided you maintain
+ clear directions next to the object code saying where to find the
+ Corresponding Source. Regardless of what server hosts the
+ Corresponding Source, you remain obligated to ensure that it is
+ available for as long as needed to satisfy these requirements.</li>
+
+<li>e) Convey the object code using peer-to-peer transmission, provided
+ you inform other peers where the object code and Corresponding
+ Source of the work are being offered to the general public at no
+ charge under subsection 6d.</li>
+
+</ul>
+
+<p>A separable portion of the object code, whose source code is excluded
+from the Corresponding Source as a System Library, need not be
+included in conveying the object code work.</p>
+
+<p>A "User Product" is either (1) a "consumer product", which means any
+tangible personal property which is normally used for personal, family,
+or household purposes, or (2) anything designed or sold for incorporation
+into a dwelling. In determining whether a product is a consumer product,
+doubtful cases shall be resolved in favor of coverage. For a particular
+product received by a particular user, "normally used" refers to a
+typical or common use of that class of product, regardless of the status
+of the particular user or of the way in which the particular user
+actually uses, or expects or is expected to use, the product. A product
+is a consumer product regardless of whether the product has substantial
+commercial, industrial or non-consumer uses, unless such uses represent
+the only significant mode of use of the product.</p>
+
+<p>"Installation Information" for a User Product means any methods,
+procedures, authorization keys, or other information required to install
+and execute modified versions of a covered work in that User Product from
+a modified version of its Corresponding Source. The information must
+suffice to ensure that the continued functioning of the modified object
+code is in no case prevented or interfered with solely because
+modification has been made.</p>
+
+<p>If you convey an object code work under this section in, or with, or
+specifically for use in, a User Product, and the conveying occurs as
+part of a transaction in which the right of possession and use of the
+User Product is transferred to the recipient in perpetuity or for a
+fixed term (regardless of how the transaction is characterized), the
+Corresponding Source conveyed under this section must be accompanied
+by the Installation Information. But this requirement does not apply
+if neither you nor any third party retains the ability to install
+modified object code on the User Product (for example, the work has
+been installed in ROM).</p>
+
+<p>The requirement to provide Installation Information does not include a
+requirement to continue to provide support service, warranty, or updates
+for a work that has been modified or installed by the recipient, or for
+the User Product in which it has been modified or installed. Access to a
+network may be denied when the modification itself materially and
+adversely affects the operation of the network or violates the rules and
+protocols for communication across the network.</p>
+
+<p>Corresponding Source conveyed, and Installation Information provided,
+in accord with this section must be in a format that is publicly
+documented (and with an implementation available to the public in
+source code form), and must require no special password or key for
+unpacking, reading or copying.</p>
+
+<h4><a name="section7"></a>7. Additional Terms.</h4>
+
+<p>"Additional permissions" are terms that supplement the terms of this
+License by making exceptions from one or more of its conditions.
+Additional permissions that are applicable to the entire Program shall
+be treated as though they were included in this License, to the extent
+that they are valid under applicable law. If additional permissions
+apply only to part of the Program, that part may be used separately
+under those permissions, but the entire Program remains governed by
+this License without regard to the additional permissions.</p>
+
+<p>When you convey a copy of a covered work, you may at your option
+remove any additional permissions from that copy, or from any part of
+it. (Additional permissions may be written to require their own
+removal in certain cases when you modify the work.) You may place
+additional permissions on material, added by you to a covered work,
+for which you have or can give appropriate copyright permission.</p>
+
+<p>Notwithstanding any other provision of this License, for material you
+add to a covered work, you may (if authorized by the copyright holders of
+that material) supplement the terms of this License with terms:</p>
+
+<ul>
+
+<li>a) Disclaiming warranty or limiting liability differently from the
+ terms of sections 15 and 16 of this License; or</li>
+
+<li>b) Requiring preservation of specified reasonable legal notices or
+ author attributions in that material or in the Appropriate Legal
+ Notices displayed by works containing it; or</li>
+
+<li>c) Prohibiting misrepresentation of the origin of that material, or
+ requiring that modified versions of such material be marked in
+ reasonable ways as different from the original version; or</li>
+
+<li>d) Limiting the use for publicity purposes of names of licensors or
+ authors of the material; or</li>
+
+<li>e) Declining to grant rights under trademark law for use of some
+ trade names, trademarks, or service marks; or</li>
+
+<li>f) Requiring indemnification of licensors and authors of that
+ material by anyone who conveys the material (or modified versions of
+ it) with contractual assumptions of liability to the recipient, for
+ any liability that these contractual assumptions directly impose on
+ those licensors and authors.</li>
+
+</ul>
+
+<p>All other non-permissive additional terms are considered "further
+restrictions" within the meaning of section 10. If the Program as you
+received it, or any part of it, contains a notice stating that it is
+governed by this License along with a term that is a further restriction,
+you may remove that term. If a license document contains a further
+restriction but permits relicensing or conveying under this License, you
+may add to a covered work material governed by the terms of that license
+document, provided that the further restriction does not survive such
+relicensing or conveying.</p>
+
+<p>If you add terms to a covered work in accord with this section, you
+must place, in the relevant source files, a statement of the
+additional terms that apply to those files, or a notice indicating
+where to find the applicable terms.</p>
+
+<p>Additional terms, permissive or non-permissive, may be stated in the
+form of a separately written license, or stated as exceptions;
+the above requirements apply either way.</p>
+
+<h4><a name="section8"></a>8. Termination.</h4>
+
+<p>You may not propagate or modify a covered work except as expressly
+provided under this License. Any attempt otherwise to propagate or
+modify it is void, and will automatically terminate your rights under
+this License (including any patent licenses granted under the third
+paragraph of section 11).</p>
+
+<p>However, if you cease all violation of this License, then your
+license from a particular copyright holder is reinstated (a)
+provisionally, unless and until the copyright holder explicitly and
+finally terminates your license, and (b) permanently, if the copyright
+holder fails to notify you of the violation by some reasonable means
+prior to 60 days after the cessation.</p>
+
+<p>Moreover, your license from a particular copyright holder is
+reinstated permanently if the copyright holder notifies you of the
+violation by some reasonable means, this is the first time you have
+received notice of violation of this License (for any work) from that
+copyright holder, and you cure the violation prior to 30 days after
+your receipt of the notice.</p>
+
+<p>Termination of your rights under this section does not terminate the
+licenses of parties who have received copies or rights from you under
+this License. If your rights have been terminated and not permanently
+reinstated, you do not qualify to receive new licenses for the same
+material under section 10.</p>
+
+<h4><a name="section9"></a>9. Acceptance Not Required for Having Copies.</h4>
+
+<p>You are not required to accept this License in order to receive or
+run a copy of the Program. Ancillary propagation of a covered work
+occurring solely as a consequence of using peer-to-peer transmission
+to receive a copy likewise does not require acceptance. However,
+nothing other than this License grants you permission to propagate or
+modify any covered work. These actions infringe copyright if you do
+not accept this License. Therefore, by modifying or propagating a
+covered work, you indicate your acceptance of this License to do so.</p>
+
+<h4><a name="section10"></a>10. Automatic Licensing of Downstream Recipients.</h4>
+
+<p>Each time you convey a covered work, the recipient automatically
+receives a license from the original licensors, to run, modify and
+propagate that work, subject to this License. You are not responsible
+for enforcing compliance by third parties with this License.</p>
+
+<p>An "entity transaction" is a transaction transferring control of an
+organization, or substantially all assets of one, or subdividing an
+organization, or merging organizations. If propagation of a covered
+work results from an entity transaction, each party to that
+transaction who receives a copy of the work also receives whatever
+licenses to the work the party's predecessor in interest had or could
+give under the previous paragraph, plus a right to possession of the
+Corresponding Source of the work from the predecessor in interest, if
+the predecessor has it or can get it with reasonable efforts.</p>
+
+<p>You may not impose any further restrictions on the exercise of the
+rights granted or affirmed under this License. For example, you may
+not impose a license fee, royalty, or other charge for exercise of
+rights granted under this License, and you may not initiate litigation
+(including a cross-claim or counterclaim in a lawsuit) alleging that
+any patent claim is infringed by making, using, selling, offering for
+sale, or importing the Program or any portion of it.</p>
+
+<h4><a name="section11"></a>11. Patents.</h4>
+
+<p>A "contributor" is a copyright holder who authorizes use under this
+License of the Program or a work on which the Program is based. The
+work thus licensed is called the contributor's "contributor version".</p>
+
+<p>A contributor's "essential patent claims" are all patent claims
+owned or controlled by the contributor, whether already acquired or
+hereafter acquired, that would be infringed by some manner, permitted
+by this License, of making, using, or selling its contributor version,
+but do not include claims that would be infringed only as a
+consequence of further modification of the contributor version. For
+purposes of this definition, "control" includes the right to grant
+patent sublicenses in a manner consistent with the requirements of
+this License.</p>
+
+<p>Each contributor grants you a non-exclusive, worldwide, royalty-free
+patent license under the contributor's essential patent claims, to
+make, use, sell, offer for sale, import and otherwise run, modify and
+propagate the contents of its contributor version.</p>
+
+<p>In the following three paragraphs, a "patent license" is any express
+agreement or commitment, however denominated, not to enforce a patent
+(such as an express permission to practice a patent or covenant not to
+sue for patent infringement). To "grant" such a patent license to a
+party means to make such an agreement or commitment not to enforce a
+patent against the party.</p>
+
+<p>If you convey a covered work, knowingly relying on a patent license,
+and the Corresponding Source of the work is not available for anyone
+to copy, free of charge and under the terms of this License, through a
+publicly available network server or other readily accessible means,
+then you must either (1) cause the Corresponding Source to be so
+available, or (2) arrange to deprive yourself of the benefit of the
+patent license for this particular work, or (3) arrange, in a manner
+consistent with the requirements of this License, to extend the patent
+license to downstream recipients. "Knowingly relying" means you have
+actual knowledge that, but for the patent license, your conveying the
+covered work in a country, or your recipient's use of the covered work
+in a country, would infringe one or more identifiable patents in that
+country that you have reason to believe are valid.</p>
+
+<p>If, pursuant to or in connection with a single transaction or
+arrangement, you convey, or propagate by procuring conveyance of, a
+covered work, and grant a patent license to some of the parties
+receiving the covered work authorizing them to use, propagate, modify
+or convey a specific copy of the covered work, then the patent license
+you grant is automatically extended to all recipients of the covered
+work and works based on it.</p>
+
+<p>A patent license is "discriminatory" if it does not include within
+the scope of its coverage, prohibits the exercise of, or is
+conditioned on the non-exercise of one or more of the rights that are
+specifically granted under this License. You may not convey a covered
+work if you are a party to an arrangement with a third party that is
+in the business of distributing software, under which you make payment
+to the third party based on the extent of your activity of conveying
+the work, and under which the third party grants, to any of the
+parties who would receive the covered work from you, a discriminatory
+patent license (a) in connection with copies of the covered work
+conveyed by you (or copies made from those copies), or (b) primarily
+for and in connection with specific products or compilations that
+contain the covered work, unless you entered into that arrangement,
+or that patent license was granted, prior to 28 March 2007.</p>
+
+<p>Nothing in this License shall be construed as excluding or limiting
+any implied license or other defenses to infringement that may
+otherwise be available to you under applicable patent law.</p>
+
+<h4><a name="section12"></a>12. No Surrender of Others' Freedom.</h4>
+
+<p>If conditions are imposed on you (whether by court order, agreement or
+otherwise) that contradict the conditions of this License, they do not
+excuse you from the conditions of this License. If you cannot convey a
+covered work so as to satisfy simultaneously your obligations under this
+License and any other pertinent obligations, then as a consequence you may
+not convey it at all. For example, if you agree to terms that obligate you
+to collect a royalty for further conveying from those to whom you convey
+the Program, the only way you could satisfy both those terms and this
+License would be to refrain entirely from conveying the Program.</p>
+
+<h4><a name="section13"></a>13. Remote Network Interaction; Use with the GNU General Public License.</h4>
+
+<p>Notwithstanding any other provision of this License, if you modify the
+Program, your modified version must prominently offer all users
+interacting with it remotely through a computer network (if your version
+supports such interaction) an opportunity to receive the Corresponding
+Source of your version by providing access to the Corresponding Source
+from a network server at no charge, through some standard or customary
+means of facilitating copying of software. This Corresponding Source
+shall include the Corresponding Source for any work covered by version 3
+of the GNU General Public License that is incorporated pursuant to the
+following paragraph.</p>
+
+<p>Notwithstanding any other provision of this License, you have permission
+to link or combine any covered work with a work licensed under version 3
+of the GNU General Public License into a single combined work, and to
+convey the resulting work. The terms of this License will continue to
+apply to the part which is the covered work, but the work with which it is
+combined will remain governed by version 3 of the GNU General Public
+License.</p>
+
+<h4><a name="section14"></a>14. Revised Versions of this License.</h4>
+
+<p>The Free Software Foundation may publish revised and/or new versions of
+the GNU Affero General Public License from time to time. Such new
+versions will be similar in spirit to the present version, but may differ
+in detail to address new problems or concerns.</p>
+
+<p>Each version is given a distinguishing version number. If the
+Program specifies that a certain numbered version of the GNU Affero
+General Public License "or any later version" applies to it, you have
+the option of following the terms and conditions either of that
+numbered version or of any later version published by the Free
+Software Foundation. If the Program does not specify a version number
+of the GNU Affero General Public License, you may choose any version
+ever published by the Free Software Foundation.</p>
+
+<p>If the Program specifies that a proxy can decide which future
+versions of the GNU Affero General Public License can be used, that
+proxy's public statement of acceptance of a version permanently
+authorizes you to choose that version for the Program.</p>
+
+<p>Later license versions may give you additional or different
+permissions. However, no additional obligations are imposed on any
+author or copyright holder as a result of your choosing to follow a
+later version.</p>
+
+<h4><a name="section15"></a>15. Disclaimer of Warranty.</h4>
+
+<p>THERE IS NO WARRANTY FOR THE PROGRAM, TO THE EXTENT PERMITTED BY
+APPLICABLE LAW. EXCEPT WHEN OTHERWISE STATED IN WRITING THE COPYRIGHT
+HOLDERS AND/OR OTHER PARTIES PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY
+OF ANY KIND, EITHER EXPRESSED OR IMPLIED, INCLUDING, BUT NOT LIMITED TO,
+THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+PURPOSE. THE ENTIRE RISK AS TO THE QUALITY AND PERFORMANCE OF THE PROGRAM
+IS WITH YOU. SHOULD THE PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF
+ALL NECESSARY SERVICING, REPAIR OR CORRECTION.</p>
+
+<h4><a name="section16"></a>16. Limitation of Liability.</h4>
+
+<p>IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING
+WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MODIFIES AND/OR CONVEYS
+THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES, INCLUDING ANY
+GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING OUT OF THE
+USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED TO LOSS OF
+DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY YOU OR THIRD
+PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER PROGRAMS),
+EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE POSSIBILITY OF
+SUCH DAMAGES.</p>
+
+<h4><a name="section17"></a>17. Interpretation of Sections 15 and 16.</h4>
+
+<p>If the disclaimer of warranty and limitation of liability provided
+above cannot be given local legal effect according to their terms,
+reviewing courts shall apply local law that most closely approximates
+an absolute waiver of all civil liability in connection with the
+Program, unless a warranty or assumption of liability accompanies a
+copy of the Program in return for a fee.</p>
+
+<p>END OF TERMS AND CONDITIONS</p>
+
+<h3><a name="howto"></a>How to Apply These Terms to Your New Programs</h3>
+
+<p>If you develop a new program, and you want it to be of the greatest
+possible use to the public, the best way to achieve this is to make it
+free software which everyone can redistribute and change under these terms.</p>
+
+<p>To do so, attach the following notices to the program. It is safest
+to attach them to the start of each source file to most effectively
+state the exclusion of warranty; and each file should have at least
+the "copyright" line and a pointer to where the full notice is found.</p>
+
+<pre> <one line to give the program's name and a brief idea of what it does.>
+ Copyright (C) <year> <name of author>
+
+ This program is free software: you can redistribute it and/or modify
+ it under the terms of the GNU Affero General Public License as
+ published by the Free Software Foundation, either version 3 of the
+ License, or (at your option) any later version.
+
+ This program is distributed in the hope that it will be useful,
+ but WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ GNU Affero General Public License for more details.
+
+ You should have received a copy of the GNU Affero General Public License
+ along with this program. If not, see <http://www.gnu.org/licenses/>.
+</pre>
+
+<p>Also add information on how to contact you by electronic and paper mail.</p>
+
+<p>If your software can interact with users remotely through a computer
+network, you should also make sure that it provides a way for users to
+get its source. For example, if your program is a web application, its
+interface could display a "Source" link that leads users to an archive
+of the code. There are many ways you could offer source, and different
+solutions will be better for different programs; see section 13 for the
+specific requirements.</p>
+
+<p>You should also get your employer (if you work as a programmer) or school,
+if any, to sign a "copyright disclaimer" for the program, if necessary.
+For more information on this, and how to apply and follow the GNU AGPL, see
+<<a href="http://www.gnu.org/licenses/">http://www.gnu.org/licenses/</a>>.</p>
+
+
--- /dev/null
+---
+layout: default
+navsection: userguide
+title: "Creative Commons"
+...
+
+<div id="deed" class="green">
+ <div id="deed-head">
+
+ <div id="deed-license">
+ <h2>Attribution-ShareAlike 3.0 United States</h2>
+ </div>
+ </div>
+
+ <h3><em>License</em></h3>
+
+ <p>THE WORK (AS DEFINED BELOW) IS PROVIDED UNDER THE TERMS
+ OF THIS CREATIVE COMMONS PUBLIC LICENSE ("CCPL" OR
+ "LICENSE"). THE WORK IS PROTECTED BY COPYRIGHT AND/OR OTHER
+ APPLICABLE LAW. ANY USE OF THE WORK OTHER THAN AS
+ AUTHORIZED UNDER THIS LICENSE OR COPYRIGHT LAW IS
+ PROHIBITED.</p>
+
+ <p>BY EXERCISING ANY RIGHTS TO THE WORK PROVIDED HERE, YOU
+ ACCEPT AND AGREE TO BE BOUND BY THE TERMS OF THIS LICENSE.
+ TO THE EXTENT THIS LICENSE MAY BE CONSIDERED TO BE A
+ CONTRACT, THE LICENSOR GRANTS YOU THE RIGHTS CONTAINED HERE
+ IN CONSIDERATION OF YOUR ACCEPTANCE OF SUCH TERMS AND
+ CONDITIONS.</p>
+
+ <p><strong>1. Definitions</strong></p>
+
+ <ol type="a">
+ <li><strong>"Collective Work"</strong> means a work, such
+ as a periodical issue, anthology or encyclopedia, in
+ which the Work in its entirety in unmodified form, along
+ with one or more other contributions, constituting
+ separate and independent works in themselves, are
+ assembled into a collective whole. A work that
+ constitutes a Collective Work will not be considered a
+ Derivative Work (as defined below) for the purposes of
+ this License.</li>
+
+ <li><strong>"Creative Commons Compatible
+ License"</strong> means a license that is listed at
+ http://creativecommons.org/compatiblelicenses that has
+ been approved by Creative Commons as being essentially
+ equivalent to this License, including, at a minimum,
+ because that license: (i) contains terms that have the
+ same purpose, meaning and effect as the License Elements
+ of this License; and, (ii) explicitly permits the
+ relicensing of derivatives of works made available under
+ that license under this License or either a Creative
+ Commons unported license or a Creative Commons
+ jurisdiction license with the same License Elements as
+ this License.</li>
+
+ <li><strong>"Derivative Work"</strong> means a work based
+ upon the Work or upon the Work and other pre-existing
+ works, such as a translation, musical arrangement,
+ dramatization, fictionalization, motion picture version,
+ sound recording, art reproduction, abridgment,
+ condensation, or any other form in which the Work may be
+ recast, transformed, or adapted, except that a work that
+ constitutes a Collective Work will not be considered a
+ Derivative Work for the purpose of this License. For the
+ avoidance of doubt, where the Work is a musical
+ composition or sound recording, the synchronization of
+ the Work in timed-relation with a moving image
+ ("synching") will be considered a Derivative Work for the
+ purpose of this License.</li>
+
+ <li><strong>"License Elements"</strong> means the
+ following high-level license attributes as selected by
+ Licensor and indicated in the title of this License:
+ Attribution, ShareAlike.</li>
+
+ <li><strong>"Licensor"</strong> means the individual,
+ individuals, entity or entities that offers the Work
+ under the terms of this License.</li>
+
+ <li><strong>"Original Author"</strong> means the
+ individual, individuals, entity or entities who created
+ the Work.</li>
+
+ <li><strong>"Work"</strong> means the copyrightable work
+ of authorship offered under the terms of this
+ License.</li>
+
+ <li><strong>"You"</strong> means an individual or entity
+ exercising rights under this License who has not
+ previously violated the terms of this License with
+ respect to the Work, or who has received express
+ permission from the Licensor to exercise rights under
+ this License despite a previous violation.</li>
+ </ol>
+
+ <p><strong>2. Fair Use Rights.</strong> Nothing in this
+ license is intended to reduce, limit, or restrict any
+ rights arising from fair use, first sale or other
+ limitations on the exclusive rights of the copyright owner
+ under copyright law or other applicable laws.</p>
+
+ <p><strong>3. License Grant.</strong> Subject to the terms
+ and conditions of this License, Licensor hereby grants You
+ a worldwide, royalty-free, non-exclusive, perpetual (for
+ the duration of the applicable copyright) license to
+ exercise the rights in the Work as stated below:</p>
+
+ <ol type="a">
+ <li>to reproduce the Work, to incorporate the Work into
+ one or more Collective Works, and to reproduce the Work
+ as incorporated in the Collective Works;</li>
+
+ <li>to create and reproduce Derivative Works provided
+ that any such Derivative Work, including any translation
+ in any medium, takes reasonable steps to clearly label,
+ demarcate or otherwise identify that changes were made to
+ the original Work. For example, a translation could be
+ marked "The original work was translated from English to
+ Spanish," or a modification could indicate "The original
+ work has been modified.";</li>
+
+ <li>to distribute copies or phonorecords of, display
+ publicly, perform publicly, and perform publicly by means
+ of a digital audio transmission the Work including as
+ incorporated in Collective Works;</li>
+
+ <li>to distribute copies or phonorecords of, display
+ publicly, perform publicly, and perform publicly by means
+ of a digital audio transmission Derivative Works.</li>
+
+ <li>
+ <p>For the avoidance of doubt, where the Work is a
+ musical composition:</p>
+
+ <ol type="i">
+ <li><strong>Performance Royalties Under Blanket
+ Licenses</strong>. Licensor waives the exclusive
+ right to collect, whether individually or, in the
+ event that Licensor is a member of a performance
+ rights society (e.g. ASCAP, BMI, SESAC), via that
+ society, royalties for the public performance or
+ public digital performance (e.g. webcast) of the
+ Work.</li>
+
+ <li><strong>Mechanical Rights and Statutory
+ Royalties</strong>. Licensor waives the exclusive
+ right to collect, whether individually or via a music
+ rights agency or designated agent (e.g. Harry Fox
+ Agency), royalties for any phonorecord You create
+ from the Work ("cover version") and distribute,
+ subject to the compulsory license created by 17 USC
+ Section 115 of the US Copyright Act (or the
+ equivalent in other jurisdictions).</li>
+ </ol>
+ </li>
+
+ <li><strong>Webcasting Rights and Statutory
+ Royalties</strong>. For the avoidance of doubt, where the
+ Work is a sound recording, Licensor waives the exclusive
+ right to collect, whether individually or via a
+ performance-rights society (e.g. SoundExchange),
+ royalties for the public digital performance (e.g.
+ webcast) of the Work, subject to the compulsory license
+ created by 17 USC Section 114 of the US Copyright Act (or
+ the equivalent in other jurisdictions).</li>
+ </ol>
+
+ <p>The above rights may be exercised in all media and
+ formats whether now known or hereafter devised. The above
+ rights include the right to make such modifications as are
+ technically necessary to exercise the rights in other media
+ and formats. All rights not expressly granted by Licensor
+ are hereby reserved.</p>
+
+ <p><strong>4. Restrictions.</strong> The license granted in
+ Section 3 above is expressly made subject to and limited by
+ the following restrictions:</p>
+
+ <ol type="a">
+ <li>You may distribute, publicly display, publicly
+ perform, or publicly digitally perform the Work only
+ under the terms of this License, and You must include a
+ copy of, or the Uniform Resource Identifier for, this
+ License with every copy or phonorecord of the Work You
+ distribute, publicly display, publicly perform, or
+ publicly digitally perform. You may not offer or impose
+ any terms on the Work that restrict the terms of this
+ License or the ability of a recipient of the Work to
+ exercise of the rights granted to that recipient under
+ the terms of the License. You may not sublicense the
+ Work. You must keep intact all notices that refer to this
+ License and to the disclaimer of warranties. When You
+ distribute, publicly display, publicly perform, or
+ publicly digitally perform the Work, You may not impose
+ any technological measures on the Work that restrict the
+ ability of a recipient of the Work from You to exercise
+ of the rights granted to that recipient under the terms
+ of the License. This Section 4(a) applies to the Work as
+ incorporated in a Collective Work, but this does not
+ require the Collective Work apart from the Work itself to
+ be made subject to the terms of this License. If You
+ create a Collective Work, upon notice from any Licensor
+ You must, to the extent practicable, remove from the
+ Collective Work any credit as required by Section 4(c),
+ as requested. If You create a Derivative Work, upon
+ notice from any Licensor You must, to the extent
+ practicable, remove from the Derivative Work any credit
+ as required by Section 4(c), as requested.</li>
+
+ <li>You may distribute, publicly display, publicly
+ perform, or publicly digitally perform a Derivative Work
+ only under: (i) the terms of this License; (ii) a later
+ version of this License with the same License Elements as
+ this License; (iii) either the Creative Commons
+ (Unported) license or a Creative Commons jurisdiction
+ license (either this or a later license version) that
+ contains the same License Elements as this License (e.g.
+ Attribution-ShareAlike 3.0 (Unported)); (iv) a Creative
+ Commons Compatible License. If you license the Derivative
+ Work under one of the licenses mentioned in (iv), you
+ must comply with the terms of that license. If you
+ license the Derivative Work under the terms of any of the
+ licenses mentioned in (i), (ii) or (iii) (the "Applicable
+ License"), you must comply with the terms of the
+ Applicable License generally and with the following
+ provisions: (I) You must include a copy of, or the
+ Uniform Resource Identifier for, the Applicable License
+ with every copy or phonorecord of each Derivative Work
+ You distribute, publicly display, publicly perform, or
+ publicly digitally perform; (II) You may not offer or
+ impose any terms on the Derivative Works that restrict
+ the terms of the Applicable License or the ability of a
+ recipient of the Work to exercise the rights granted to
+ that recipient under the terms of the Applicable License;
+ (III) You must keep intact all notices that refer to the
+ Applicable License and to the disclaimer of warranties;
+ and, (IV) when You distribute, publicly display, publicly
+ perform, or publicly digitally perform the Work, You may
+ not impose any technological measures on the Derivative
+ Work that restrict the ability of a recipient of the
+ Derivative Work from You to exercise the rights granted
+ to that recipient under the terms of the Applicable
+ License. This Section 4(b) applies to the Derivative Work
+ as incorporated in a Collective Work, but this does not
+ require the Collective Work apart from the Derivative
+ Work itself to be made subject to the terms of the
+ Applicable License.</li>
+
+ <li>If You distribute, publicly display, publicly
+ perform, or publicly digitally perform the Work (as
+ defined in Section 1 above) or any Derivative Works (as
+ defined in Section 1 above) or Collective Works (as
+ defined in Section 1 above), You must, unless a request
+ has been made pursuant to Section 4(a), keep intact all
+ copyright notices for the Work and provide, reasonable to
+ the medium or means You are utilizing: (i) the name of
+ the Original Author (or pseudonym, if applicable) if
+ supplied, and/or (ii) if the Original Author and/or
+ Licensor designate another party or parties (e.g. a
+ sponsor institute, publishing entity, journal) for
+ attribution ("Attribution Parties") in Licensor's
+ copyright notice, terms of service or by other reasonable
+ means, the name of such party or parties; the title of
+ the Work if supplied; to the extent reasonably
+ practicable, the Uniform Resource Identifier, if any,
+ that Licensor specifies to be associated with the Work,
+ unless such URI does not refer to the copyright notice or
+ licensing information for the Work; and, consistent with
+ Section 3(b) in the case of a Derivative Work, a credit
+ identifying the use of the Work in the Derivative Work
+ (e.g., "French translation of the Work by Original
+ Author," or "Screenplay based on original Work by
+ Original Author"). The credit required by this Section
+ 4(c) may be implemented in any reasonable manner;
+ provided, however, that in the case of a Derivative Work
+ or Collective Work, at a minimum such credit will appear,
+ if a credit for all contributing authors of the
+ Derivative Work or Collective Work appears, then as part
+ of these credits and in a manner at least as prominent as
+ the credits for the other contributing authors. For the
+ avoidance of doubt, You may only use the credit required
+ by this Section for the purpose of attribution in the
+ manner set out above and, by exercising Your rights under
+ this License, You may not implicitly or explicitly assert
+ or imply any connection with, sponsorship or endorsement
+ by the Original Author, Licensor and/or Attribution
+ Parties, as appropriate, of You or Your use of the Work,
+ without the separate, express prior written permission of
+ the Original Author, Licensor and/or Attribution
+ Parties.</li>
+ </ol>
+
+ <p><strong>5. Representations, Warranties and
+ Disclaimer</strong></p>
+
+ <p>UNLESS OTHERWISE MUTUALLY AGREED TO BY THE PARTIES IN
+ WRITING, LICENSOR OFFERS THE WORK AS-IS AND ONLY TO THE
+ EXTENT OF ANY RIGHTS HELD IN THE LICENSED WORK BY THE
+ LICENSOR. THE LICENSOR MAKES NO REPRESENTATIONS OR
+ WARRANTIES OF ANY KIND CONCERNING THE WORK, EXPRESS,
+ IMPLIED, STATUTORY OR OTHERWISE, INCLUDING, WITHOUT
+ LIMITATION, WARRANTIES OF TITLE, MARKETABILITY,
+ MERCHANTIBILITY, FITNESS FOR A PARTICULAR PURPOSE,
+ NONINFRINGEMENT, OR THE ABSENCE OF LATENT OR OTHER DEFECTS,
+ ACCURACY, OR THE PRESENCE OF ABSENCE OF ERRORS, WHETHER OR
+ NOT DISCOVERABLE. SOME JURISDICTIONS DO NOT ALLOW THE
+ EXCLUSION OF IMPLIED WARRANTIES, SO SUCH EXCLUSION MAY NOT
+ APPLY TO YOU.</p>
+
+ <p><strong>6. Limitation on Liability.</strong> EXCEPT TO
+ THE EXTENT REQUIRED BY APPLICABLE LAW, IN NO EVENT WILL
+ LICENSOR BE LIABLE TO YOU ON ANY LEGAL THEORY FOR ANY
+ SPECIAL, INCIDENTAL, CONSEQUENTIAL, PUNITIVE OR EXEMPLARY
+ DAMAGES ARISING OUT OF THIS LICENSE OR THE USE OF THE WORK,
+ EVEN IF LICENSOR HAS BEEN ADVISED OF THE POSSIBILITY OF
+ SUCH DAMAGES.</p>
+
+ <p><strong>7. Termination</strong></p>
+
+ <ol type="a">
+ <li>This License and the rights granted hereunder will
+ terminate automatically upon any breach by You of the
+ terms of this License. Individuals or entities who have
+ received Derivative Works or Collective Works from You
+ under this License, however, will not have their licenses
+ terminated provided such individuals or entities remain
+ in full compliance with those licenses. Sections 1, 2, 5,
+ 6, 7, and 8 will survive any termination of this
+ License.</li>
+
+ <li>Subject to the above terms and conditions, the
+ license granted here is perpetual (for the duration of
+ the applicable copyright in the Work). Notwithstanding
+ the above, Licensor reserves the right to release the
+ Work under different license terms or to stop
+ distributing the Work at any time; provided, however that
+ any such election will not serve to withdraw this License
+ (or any other license that has been, or is required to
+ be, granted under the terms of this License), and this
+ License will continue in full force and effect unless
+ terminated as stated above.</li>
+ </ol>
+
+ <p><strong>8. Miscellaneous</strong></p>
+
+ <ol type="a">
+ <li>Each time You distribute or publicly digitally
+ perform the Work (as defined in Section 1 above) or a
+ Collective Work (as defined in Section 1 above), the
+ Licensor offers to the recipient a license to the Work on
+ the same terms and conditions as the license granted to
+ You under this License.</li>
+
+ <li>Each time You distribute or publicly digitally
+ perform a Derivative Work, Licensor offers to the
+ recipient a license to the original Work on the same
+ terms and conditions as the license granted to You under
+ this License.</li>
+
+ <li>If any provision of this License is invalid or
+ unenforceable under applicable law, it shall not affect
+ the validity or enforceability of the remainder of the
+ terms of this License, and without further action by the
+ parties to this agreement, such provision shall be
+ reformed to the minimum extent necessary to make such
+ provision valid and enforceable.</li>
+
+ <li>No term or provision of this License shall be deemed
+ waived and no breach consented to unless such waiver or
+ consent shall be in writing and signed by the party to be
+ charged with such waiver or consent.</li>
+
+ <li>This License constitutes the entire agreement between
+ the parties with respect to the Work licensed here. There
+ are no understandings, agreements or representations with
+ respect to the Work not specified here. Licensor shall
+ not be bound by any additional provisions that may appear
+ in any communication from You. This License may not be
+ modified without the mutual written agreement of the
+ Licensor and You.</li>
+ </ol>
+ <!-- BREAKOUT FOR CC NOTICE. NOT A PART OF THE LICENSE -->
+
+ <blockquote>
+ <h3>Creative Commons Notice</h3>
+
+ <p>Creative Commons is not a party to this License, and
+ makes no warranty whatsoever in connection with the Work.
+ Creative Commons will not be liable to You or any party
+ on any legal theory for any damages whatsoever, including
+ without limitation any general, special, incidental or
+ consequential damages arising in connection to this
+ license. Notwithstanding the foregoing two (2) sentences,
+ if Creative Commons has expressly identified itself as
+ the Licensor hereunder, it shall have all rights and
+ obligations of Licensor.</p>
+
+ <p>Except for the limited purpose of indicating to the
+ public that the Work is licensed under the CCPL, Creative
+ Commons does not authorize the use by either party of the
+ trademark "Creative Commons" or any related trademark or
+ logo of Creative Commons without the prior written
+ consent of Creative Commons. Any permitted use will be in
+ compliance with Creative Commons' then-current trademark
+ usage guidelines, as may be published on its website or
+ otherwise made available upon request from time to time.
+ For the avoidance of doubt, this trademark restriction
+ does not form part of this License.</p>
+
+ <p>Creative Commons may be contacted at <a href=
+ "http://creativecommons.org/">http://creativecommons.org/</a>.</p>
+ </blockquote>
+ </div>
+ </div>
+
+ </div>
--- /dev/null
+---
+layout: default
+navsection: userguide
+title: "Arvados Free Software Licenses"
+...
+
+Server-side components of Arvados contained in the apps/ and services/ directories, including the API Server, Workbench, and Crunch, are licenced under the "GNU Affero General Public License version 3":agpl-3.0.html.
+
+The Arvados client Software Development Kits contained in the sdk/ directory, example scripts in the crunch_scripts/ directory, and code samples in the Aravados documentation are licensed under the "Apache License, Version 2.0":LICENSE-2.0.html
+
+The Arvados Documentation located in the doc/ directory is licensed under the "Creative Commons Attribution-Share Alike 3.0 United States":by-sa-3.0.html
---
layout: default
navsection: userguide
-navmenu: Examples
title: "Crunch examples"
-
...
-h1. Crunch examples
-
Several crunch scripts are included with Arvados in the "/crunch_scripts directory":https://arvados.org/projects/arvados/repository/revisions/master/show/crunch_scripts. They are intended to provide examples and starting points for writing your own scripts.
h4. bwa-aln
---
layout: default
navsection: userguide
-navmenu: Getting Started
title: "Checking your environment"
-
...
-h1. Checking your environment
-
First you should "log into an Arvados VM instance":{{site.baseurl}}/user/getting_started/ssh-access.html#login if you have not already done so.
If @arv user current@ is able to access the API server, it will print out information about your account. Check that you are able to access the Arvados API server using the following command:
bc. ARVADOS_API_HOST and ARVADOS_API_TOKEN need to be defined as environment variables
Then follow the instructions for "getting an API token,":{{site.baseurl}}/user/reference/api-tokens.html and try @arv user current@ again.
-
-Once you are able to access the API server, you are ready proceed to the first tutorial: "Storing and retrieving data using Arvados Keep.":{{site.baseurl}}/user/tutorials/tutorial-keep.html
---
layout: default
navsection: userguide
-navmenu: Getting Started
title: Arvados Community and Getting Help
-
...
-h1. Arvados Community and Getting Help
-
h2. On the web
---
layout: default
navsection: userguide
-navmenu: Getting Started
title: Accessing an Arvados VM over ssh
-
...
-h1. Accessing an Arvados Virtual Machine over ssh
-
-Arvados requires a public @ssh@ key in order to securely log in to an Arvados VM instance, or to access an Arvados @git@ repository.
+Arvados requires a public ssh key in order to securely log in to an Arvados VM instance, or to access an Arvados @git@ repository.
This document is divided up into three sections.
notextile. <pre><code>$ <span class="userinput">ls ~/.ssh/id_rsa.pub</span></code></pre>
-If the file @id_rsa.pub@ exists, then you may use your existing key. Copy the contents of @~/.ssh/id_rsa.pub@ onto the clipboard (this is your public key). Proceed to "adding your key to the Arvados Workbench.":#workbench
+If the file @id_rsa.pub@ exists, then you may use your existing key. Copy the contents of @~/.ssh/id_rsa.pub@ onto the clipboard (this is your public key). You can skip this step and proceed by "adding your key to the Arvados Workbench.":#workbench
If there is no file @~/.ssh/id_rsa.pub@, you must generate a new key. Use @ssh-keygen@ to do this:
</code></pre>
</notextile>
-Now you can set up @ssh-agent@ (next) or proceed to "adding your key to the Arvados Workbench.":#workbench
+Now you can set up @ssh-agent@ (next) or proceed with "adding your key to the Arvados Workbench.":#workbench
h3. Setting up ssh-agent (recommended)
notextile. <pre><code>$ <span class="userinput">eval $(ssh-agent -s)</span></code></pre>
-* @ssh-agent -s@ prints out values for environment variables SSH_AUTH_SOCK and SSH_AGENT_PID and then runs in the background. Using "eval" on the output as shown here causes those variables to be set in the current shell environment so that subsequent calls to @ssh@ can discover how to access the @ssh-agent@ daemon.
+@ssh-agent -s@ prints out values for environment variables SSH_AUTH_SOCK and SSH_AGENT_PID and then runs in the background. Using "eval" on the output as shown here causes those variables to be set in the current shell environment so that subsequent calls to @ssh@ can discover how to access the @ssh-agent@ daemon.
-After running @ssh-agent@, or if @ssh-add -l@ prints "The agent has no identities", then you will need to add your key using the following command. The passphrase to decrypt the key is the same used to protect the key when it was created with @ssh-keygen@:
+After running @ssh-agent@, or if @ssh-add -l@ prints "The agent has no identities", then you will need to add your key using the following command. The passphrase to decrypt the key is the same used to protect the key when it was created with @ssh-keygen@:
<notextile>
<pre><code>$ <span class="userinput">ssh-add</span>
(Note: if you are using the @ssh@ client that comes with "Cygwin":http://cygwin.com you should follow the "Unix":#unix instructions).
-"PuTTY":http://www.putty.org/ is a free (MIT-licensed) Win32 Telnet and SSH client. PuTTy includes all the tools a windows user needs to set up Private Keys and to set up and use SSH connections to your virtual machines in the Arvados Cloud.
-
-You can use PuTTY to create public/private keys, which are how you’ll ensure that that access to Arvados cloud is secure. You can also use PuTTY as an SSH client to access your virtual machine in an Arvados cloud and work with the Arvados Command Line Interface (CLI) client.
+"PuTTY":http://www.chiark.greenend.org.uk/~sgtatham/putty/ is a free (MIT-licensed) Win32 Telnet and SSH client. PuTTY includes all the tools a Windows user needs to create private keys and make ssh connections to your virtual machines in the Arvados Cloud.
-You may download putty from "http://www.putty.org/":http://www.putty.org/ .
-
-Note that you should download the installer or .zip file with all of the PuTTY tools (PuTTYtel is not required).
+You can "download PuTTY from its Web site":http://www.chiark.greenend.org.uk/~sgtatham/putty/. Note that you should download the installer or .zip file with all of the PuTTY tools (PuTTYtel is not required).
h3. Step 1 - Adding PuTTY to the PATH
# Open the Control Panel.
# Select _Advanced System Settings_, and choose _Environment Variables_.
# Under system variables, find and edit @PATH@.
-# Add the following to the end of PATH (make sure to include semi colon and quotation marks):
+# If you installed PuTTY in @C:\Program Files\PuTTY\@, add the following to the end of PATH (make sure to include semicolon and quotation marks):
+<code>;\"C:\Program Files\PuTTY\"</code>
+If you installed PuTTY in @C:\Program Files (x86)\PuTTY\@, add the following to the end of PATH (make sure to include semicolon and quotation marks):
<code>;\"C:\Program Files (x86)\PuTTY\"</code>
# Click through the OKs to close all the dialogs you’ve opened.
# At the bottom of the window, make sure the ‘Number of bits in a generated key’ field is set to 4096.
# Click Generate and follow the instructions to generate a key.
# Click to save the Public Key.
-# Click to save the Private Key (we recommend using a strong passphrase) .
+# Click to save the Private Key (we recommend using a strong passphrase).
# Select the text of the Public Key and copy it to the clipboard.
h3. Step 3 - Set up Pageant
-Note: Pageant is a PuTTY utility that manages your private keys so is not necessary to enter your private key passphrase every time you need to make a new ssh connection.
+Pageant is a PuTTY utility that manages your private keys so is not necessary to enter your private key passphrase every time you make a new ssh connection.
# Start Pageant from the Start Menu or the folder where it was installed.
# Pageant will now be running in the system tray. Click the Pageant icon to configure.
# Choose _Add Key_ and add the private key which you created in the previous step.
-You are now ready to proceed to "adding your key to the Arvados Workbench":#workbench .
-
-_Note: We recommend you do not delete the “Default” Saved Session._
+You are now ready to proceed to "adding your key to the Arvados Workbench.":#workbench
h1(#workbench). Adding your key to Arvados Workbench
-h3. From the workbench dashboard
+h3. From the Workbench dashboard
-If you have no @ssh@ keys registered, there should be a notification asking you to provide your @ssh@ public key. On the Workbench dashboard (in this guide, this is "https://workbench.{{ site.arvados_api_host }}/":https://workbench.{{ site.arvados_api_host }}/ ), look for the envelope icon <span class="glyphicon glyphicon-envelope"></span> <span class="badge badge-alert">1</span> in upper right corner (the number indicates there are new notifications). Click on this icon and a dropdown menu should appear with a message asking you to add your public key. Paste your public key into the text area provided and click on the check button to submit the key. You are now ready to "log into an Arvados VM":#login.
+If you have no ssh keys registered, there should be a notification asking you to provide your ssh public key. On the Workbench dashboard, look for the envelope icon <span class="glyphicon glyphicon-envelope"></span> <span class="badge badge-alert">1</span> in upper right corner (the number indicates there are new notifications). Click on this icon and a dropdown menu should appear with a message asking you to add your public key. Paste your public key into the text area provided and click on the check button to submit the key. You are now ready to "log into an Arvados VM":#login.
h3. Alternate way to add ssh keys
-If you want to add additional @ssh@ keys, click on the user icon <span class="glyphicon glyphicon-user"></span> in the upper right corner to access the user settings menu, and click on the menu item _Manage ssh keys_ to go to the Authorized keys page.
+If you want to add additional ssh keys, click on the user icon <span class="glyphicon glyphicon-user"></span> in the upper right corner to access the user settings menu, and click on the menu item *Manage ssh keys* to go to the Authorized keys page.
-On _Authorized keys_ page, the click on the button <span class="btn btn-primary disabled">Add a new authorized key</span> in the upper right corner.
+On the *Authorized keys* page, the click on the button <span class="btn btn-primary disabled">Add a new authorized key</span> in the upper right corner.
-The page will reload with a new row of information. Under the *public_key* column heading, click on the cell +none+ . This will open an editing popup as shown in this screenshot:
+The page will reload with a new row of information. Under the *public_key* column heading, click on the cell +none+. This will open an editing popup as shown in this screenshot:
!{{ site.baseurl }}/images/ssh-adding-public-key.png!
-Paste the public key from the previous section into the popup text box and click on the check mark to save it. This should refresh the page with the public key that you just added now listed under the *public_key* column. You are now ready to "log into an Arvados VM":#login.
+Paste the public key that you copied to the cliboard in the previous section into the popup text box, then click on the check mark to save it. This should refresh the page with the public key that you just added now listed under the *public_key* column. You are now ready to "log into an Arvados VM":#login.
h1(#login). Using ssh to log into an Arvados VM
-To see a list of virtual machines that you have access to and determine the name and login information, click on Compute %(rarr)→% Virtual machines. Once on the "virtual machines" page, The *hostname* columns lists the name of each available VM. The *logins* column will have a value in the form of @["you"]@. Ignore the square brackets and quotes to get your login name. In this guide the hostname will be _shell_ and the login will be _you_. Replace these with your hostname and login as appropriate.
+To see a list of virtual machines that you have access to and determine the name and login information, click on Compute %(rarr)→% Virtual machines. Once on the *Virtual machines* page, The *hostname* columns lists the name of each available VM. The *logins* column will have a value in the form of @["you"]@. Your login name is the text inside the quotes. In this guide the hostname will be _shell_ and the login will be _you_. Replace these with your hostname and login name as appropriate.
This section consists of two sets of instructions, depending on whether you will be logging in using a "Unix":#unixvm (Linux, OS X, Cygwin) or "Windows":#windowsvm client.
h2(#unixvm). Logging in using command line ssh (Unix)
-h3. Connecting to the VM
-
-Use the following command to connect to the "shell" VM instance as "you". Replace *<code>you@shell</code>* at the end of the following command with your *login* and *hostname* from Workbench:
+h3. Connecting to the virtual machine
-notextile. <pre><code>$ <span class="userinput">ssh -o "ProxyCommand ssh -a -x -p2222 turnout@switchyard.{{ site.arvados_api_host }} shell" -A -x <b>you@shell</b></span></code></pre>
+Use the following command to connect to the _shell_ VM instance as _you_. Replace *<code>you@shell</code>* at the end of the following command with your *login* and *hostname* from Workbench:
-There are several things going on here:
+notextile. <pre><code>$ <span class="userinput">ssh -o "ProxyCommand ssh -a -x -p2222 turnout@switchyard.{{ site.arvados_api_host }} <b>shell</b>" -A -x <b>you@shell</b></span></code></pre>
-The VMs typically have addresses that are not globally routable, so you cannot log in directly. Instead, you log into a "switchyard" server and then tell the switchyard which VM you want to connect to.
+This command does several things at once. You usually cannot log in directly to virtual machines over the public Internet. Instead, you log into a "switchyard" server and then tell the switchyard which virtual machine you want to connect to.
-* @-o "ProxyCommand ..."@ option instructs ssh to run the specified command and then tunnel your ssh connection over the proxy.
-* @-a@ tells ssh not to forward your ssh-agent credentials to the switchyard
-* @-x@ tells ssh not to forward your X session to the switchyard
-* @-p2222@ specifies that the switchyard is running on non-standard port 2222
-* <code>turnout@switchyard.{{ site.arvados_api_host }}</code> specifies the user (@turnout@) and hostname (@switchyard.{{ site.arvados_api_host }}@) of the switchboard server that will proxy our connection to the VM.
-* @shell@ is the name of the VM that we want to connect to. This is sent to the switchyard server as if it were an ssh command, and the switchyard server connects to the VM on our behalf.
-* After the ProxyCommand section, the @-x@ must be repeated because it applies to the connection to VM instead of the switchyard.
+* @-o "ProxyCommand ..."@ configures ssh to run the specified command to create a proxy and route your connection through it.
+* @-a@ tells ssh not to forward your ssh-agent credentials to the switchyard.
+* @-x@ tells ssh not to forward your X session to the switchyard.
+* @-p2222@ specifies that the switchyard is running on non-standard port 2222.
+* <code>turnout@switchyard.{{ site.arvados_api_host }}</code> specifies the user (@turnout@) and hostname (@switchyard.{{ site.arvados_api_host }}@) of the switchyard server that will proxy our connection to the VM.
+* *@shell@* is the name of the VM that we want to connect to. This is sent to the switchyard server as if it were an ssh command, and the switchyard server connects to the VM on our behalf.
+* After the ProxyCommand section, we repeat @-x@ to disable X session forwarding to the virtual machine.
* @-A@ specifies that we want to forward access to @ssh-agent@ to the VM.
-* Finally, *<code>you@shell</code>* specifies your username and repeats the hostname of the VM. The username can be found in the *logins* column in the VMs Workbench page, discussed above.
+* Finally, *<code>you@shell</code>* specifies your login name and repeats the hostname of the VM. The username can be found in the *logins* column in the VMs Workbench page, discussed in the previous section.
You should now be able to log into the Arvados VM and "check your environment.":check-environment.html
h3. Configuration (recommended)
-Since the above command line is cumbersome, it can be greatly simplfied by adding the following section your @~/.ssh/config@ file:
+The command line above is cumbersome, but you can configure ssh to remember many of these settings. Add this text to the file @.ssh/config@ in your home directory (create a new file if @.ssh/config@ doesn't exist):
<notextile>
<pre><code class="userinput">Host *.arvados
ProxyCommand ssh -a -x -p2222 turnout@switchyard.{{ site.arvados_api_host }} $SSH_PROXY_FLAGS %h
+ User <b>you</b>
ForwardAgent yes
ForwardX11 no
</code></pre>
This will recognize any host ending in ".arvados" and automatically apply the proxy, user and forwarding settings from the configuration file, allowing you to log in with a much simpler command:
-notextile. <pre><code>$ <span class="userinput">ssh <b>you@shell</b>.arvados</span></code></pre>
+notextile. <pre><code>$ <span class="userinput">ssh <b>shell</b>.arvados</span></code></pre>
h2(#windowsvm). Logging in using PuTTY (Windows)
# Open PuTTY from the Start Menu.
# On the Session screen set the Host Name (or IP address) to “shell”.
# On the Session screen set the Port to “22”.
-# On the Connection %(rarr)→% Data screen set the Auto-login username to the username listed in the *logins* column on the Arvados Workbench _Access %(rarr)→% VMs_ page.
+# On the Connection %(rarr)→% Data screen set the Auto-login username to the username listed in the *logins* column on the Arvados Workbench page _Compute %(rarr)→% Virtual machines_.
# On the Connection %(rarr)→% Proxy screen set the Proxy Type to “Local”.
# On the Connection %(rarr)→% Proxy screen in the “Telnet command, or local proxy command” box enter:
<code>plink -P 2222 turnout@switchyard.qr1hi.arvadosapi.com %host</code>
Make sure there is no newline at the end of the text entry.
-# Return to the Session screen. In the Saved Sessions box, enter a name for this configuration and hit Save.
+# Return to the Session screen. In the Saved Sessions box, enter a name for this configuration and click Save.
+
+_Note: We recommend you do not delete the “Default” Saved Session._
h3. Connecting to the VM
-# Open PuTTY
+# Open PuTTY from the Start Menu.
# Click on the Saved Session name you created in the previous section.
# Click Load to load those saved session settings.
-# Click Open and that will open the SSH window at the command prompt. You will now be logged in to your virtual machine.
+# Click Open to open the SSH window at the command prompt. You will now be logged into your virtual machine.
You should now be able to log into the Arvados VM and "check your environment.":check-environment.html
---
layout: default
navsection: userguide
-navmenu: Getting Started
title: Accessing Arvados Workbench
-
...
-h1. Accessing Arvados Workbench
Access the Arvados beta test instance available using this link:
-"https://workbench.{{ site.arvados_api_host }}/":https://workbench.{{ site.arvados_api_host }}/
+"https://{{ site.arvados_workbench_host }}/":https://{{ site.arvados_workbench_host }}/
If you are accessing Arvados for the first time, you will be asked to log in using a Google account. Arvados uses only your name and email address from Google services for identification, and will never access any personal information. Once you are logged in, the Workbench page may indicate your account status is *New / inactive*. If this is the case, contact the administrator of the Arvados instance to activate your account.
Once your account is active, logging in to the Workbench will present you with a system status dashboard. This gives a summary of data, configuration, and activity in the Arvados instance.
-
-Next, we will "configure your account for ssh access to an Arvados virtual machine (VM).":ssh-access.html
layout: default
navsection: userguide
title: Welcome to Arvados!
-
...
-h1. Welcome to Arvados!
-
-This guide is intended to introduce new users to the Arvados system. It covers initial configuration required to use the system and then presents several tutorials on using Arvados to do data processing.
+This guide is intended to introduce new users to the Arvados system. It covers initial configuration required to access the system and then presents several tutorials on using Arvados to do data processing.
This user guide introduces how to use the major components of Arvados. These are:
* Keep: Content-addressable cluster file system designed for robust storage of very large files, such as whole genome sequences running in the hundreds of gigabytes
-* Crunch: Cluster compute engine designed for genomic analysis, e.g. alignment, variant calls
-* Metadata Database: Information about the genomic data stored in Keep, such as genomic traits, human subjects
-* Workbench: Web interface to Arvados components
+* Crunch: Cluster compute engine designed for genomic analysis, such as alignment and variant calls
+* Metadata Database: Information about the genomic data stored in Keep, such as genomic traits and human subjects
+* Workbench: Arvados' Web interface
h2. Prerequisites
To get the most value out of this guide, you should be comfortable with the following:
-# Using a secure shell client such as @ssh@ or @putty@ to log on to a remote server
-# Using the unix command line shell @bash@
+# Using a secure shell client such as @ssh@ or @putty@ to log on to a remote server
+# Using the Unix command line shell @bash@
# Viewing and editing files using a unix text editor such as @vi@, @emacs@, or @nano@
# Programming in @python@
# Revision control using @git@
-The examples in this guide uses the public Arvados instance located at "https://workbench.{{ site.arvados_api_host }}/":https://workbench.{{ site.arvados_api_host }}/ . You must have an account in order to use this service. If you would like to request an account, please send an email to "arvados@curoverse.com":mailto:arvados@curoverse.com .
+We also recommend you read the "Arvados Platform Overview":https://arvados.org/projects/arvados/wiki#Platform-Overview for an introduction and background information about Arvados.
+
+The examples in this guide use the Arvados instance located at "https://{{ site.arvados_workbench_host }}/":https://{{ site.arvados_workbench_host }}/. If you are using a different Arvados instance replace @{{ site.arvados_workbench_host }}@ with your private instance in all of the examples in this guide.
-If you are using a different Arvados instance replace @{{ site.arvados_api_host }}@ with your private instance in all of the examples in this guide.
+The Arvados public beta instance is located at "https://workbench.qr1hi.arvadosapi.com/":https://workbench.qr1hi.arvadosapi.com/. You must have an account in order to use this service. If you would like to request an account, please send an email to "arvados@curoverse.com":mailto:arvados@curoverse.com.
h2. Typographic conventions
<notextile>
<ul>
-<li>Code blocks which are set aside from the text indicate user input to the system. Commands that should be entered into a Unix shell are indicated by the directory where you should enter the command ('~' indicates your home directory) followed by '$', followed by the highlighted <span class="userinput">command to enter</span> (do not enter the '$'), and possibly followed by example command output in black. For example, the following block indicates that you should type "ls foo" while in your home directory and the expected output will be "foo".
-
-<pre><code>~$ <span class="userinput">ls foo</span>
-foo
-</code></pre></li>
+<li>Code blocks which are set aside from the text indicate user input to the system. Commands that should be entered into a Unix shell are indicated by the directory where you should enter the command ('~' indicates your home directory) followed by '$', followed by the highlighted <span class="userinput">command to enter</span> (do not enter the '$'), and possibly followed by example command output in black. For example, the following block indicates that you should type <code>ls foo.*</code> while in your home directory and the expected output will be "foo.input" and "foo.output".
+<pre><code>~$ <span class="userinput">ls foo.*</span>
+foo.input foo.output
+</code></pre>
+</li>
<li>Code blocks inline with text emphasize specific <code>programs</code>, <code>files</code>, or <code>options</code> that are being discussed.</li>
-<li>Bold text emphasizes <b>specific items</b> to look when discussing Arvados Workbench pages.</li>
-<li>A sequence of steps separated by right arrows (<span class="rarr">→</span>) indicate a path the user should follow through the Arvados Workbench to access some piece of information under discussion. The steps indicate a menu, hyperlink, column name, field name, or other label on the page that guide the user where to look or click.
+<li>Bold text emphasizes <b>specific items</b> to review on Arvados Workbench pages.</li>
+<li>A sequence of steps separated by right arrows (<span class="rarr">→</span>) indicate a path the user should follow through the Arvados Workbench. The steps indicate a menu, hyperlink, column name, field name, or other label on the page that guide the user where to look or click.
</li>
</ul>
</notextile>
-Now begin by "accessing the Arvados workbench.":getting_started/workbench.html
---
layout: default
navsection: userguide
-navmenu: Reference
title: "Getting an API token"
-
...
-h1. Reference: Getting an API token
-
The Arvados API token is a secret key that enables the @arv@ command line client to access Arvados with the proper permissions.
-Access the Arvados workbench using this link: "https://workbench.{{ site.arvados_api_host }}/":https://workbench.{{ site.arvados_api_host }}/
-
-(Replace @{{ site.arvados_api_host }}@ with the hostname of your local Arvados instance if necessary.)
+Access the Arvados Workbench using this link: "https://{{ site.arvados_workbench_host }}/":https://{{ site.arvados_workbench_host }}/ (Replace @{{ site.arvados_api_host }}@ with the hostname of your local Arvados instance if necessary.)
-First, open a shell on the system on which you intend to use the Arvados client (this may be your local workstation, or an Arvados VM, refer to "Accessing Arvados over ssh":{{site.baseurl}}/user/getting_started/ssh-access.html ) .
+Open a shell on the system where you want to use the Arvados client. This may be your local workstation, or "an Arvados virtual machine accessed with ssh":{{site.baseurl}}/user/getting_started/ssh-access.html.
-Click on the user icon <span class="glyphicon glyphicon-user"></span> in the upper right corner to access the user settings menu, and click on the menu item _Manage API token_ to go to the "api client authorizations" page.
+Click on the user icon <span class="glyphicon glyphicon-user"></span> in the upper right corner to access the user settings menu. Click on the menu item *Manage API tokens* to go to the "Api client authorizations" page.
h2. The easy way
-For your convenience, the "api client authorizations" page on Workbench provides a "Help" tab that provides a command you may copy and paste directly into the shell. It will look something like this:
+For your convenience, the "Api client authorizations" page on Workbench provides a *Help* tab that includes a command you may copy and paste directly into the shell. It will look something like this:
bc. ### Pasting the following lines at a shell prompt will allow Arvados SDKs
-### to authenticate to your account, youraddress@example.com
+### to authenticate to your account, you@example.com
read ARVADOS_API_TOKEN <<EOF
2jv9346o396exampledonotuseexampledonotuseexes7j1ld
EOF
export ARVADOS_API_TOKEN ARVADOS_API_HOST={{ site.arvados_api_host }}
-* The @read@ command takes the contents of stdin and puts it into the shell variable named on the command line.
-* The @<<EOF@ notation means read each line on stdin and pipe it to the command, terminating on reading the line @EOF@.
-* The @export@ command puts a local shell variable into the environment that will be inherited by child processes (e.g. the @arv@ client).
+* The @read@ command reads text input until @EOF@ (designated by @<<EOF@) and stores it in the @ARVADOS_API_TOKEN@ environment variable.
+* The @export@ command puts a local shell variable into the environment that will be inherited by child processes such as the @arv@ client.
h2. Setting the environment manually
</code></pre>
</notextile>
-* @ARVADOS_API_HOST@ tells @arv@ which host to connect to
-* @ARVADOS_API_TOKEN@ is the secret key used by the Arvados API server to authenticate access.
+* @ARVADOS_API_HOST@ tells @arv@ which host to connect to.
+* @ARVADOS_API_TOKEN@ is the secret key used by the Arvados API server to authenticate access. Its value is the text you copied from the *api_token* column on the Workbench.
If you are connecting to a development instance with a unverified/self-signed SSL certificate, set this variable to skip SSL validation:
h2. settings.conf
-Arvados tools will also look for the authentication information in @~/.config/arvados/settings.conf@. If you have already put the variables into the environment with instructions above, you can use these commands to create an Arvados configuration file:
+Arvados tools will also look for the authentication information in @~/.config/arvados/settings.conf@. If you have already put the variables into the environment following the instructions above, you can use these commands to create an Arvados configuration file:
<notextile>
<pre><code>$ <span class="userinput">echo "ARVADOS_API_HOST=$ARVADOS_API_HOST" > ~/.config/arvados/settings.conf</span>
h2. .bashrc
-Alternately, you may add the declarations of @ARVADOS_API_HOST@ and @ARVADOS_API_TOKEN@ to the @~/.bashrc@ file on the system on which you intend to use the Arvados client. If you have already put the variables into the environment with instructions above, you can use these commands to append the environment variables to your @~/.bashrc@:
+Alternately, you may add the declarations of @ARVADOS_API_HOST@ and @ARVADOS_API_TOKEN@ to the @~/.bashrc@ file on the system on which you intend to use the Arvados client. If you have already put the variables into the environment following the instructions above, you can use these commands to append the environment variables to your @~/.bashrc@:
<notextile>
<pre><code>$ <span class="userinput">echo "export ARVADOS_API_HOST=$ARVADOS_API_HOST" >> ~/.bashrc</span>
--- /dev/null
+---
+layout: default
+navsection: userguide
+title: "Job and Pipeline Reference"
+...
+
+h2. Submitting jobs
+
+table(table table-bordered table-condensed).
+|_. Attribute |_. Type|_. Accepted values |_. Required|_. Description|
+|script |string |filename |yes |The actual script that will be run by crunch. Must be the name of an executable file in the crunch_scripts/ directory at the git revision specified by script_version.|
+|script_version |string |git branch, tag, or version hash |yes |The code version to run, which is available in the specified repository. May be a git hash or tag to specify an exact version, or a branch. If it is a branch, use the branch head.|
+|repository |string |name of git repository hosted by Arvados |yes |The repository to search for script_version.|
+|script_parameters |object |any JSON object |yes |The input parameters for the job, with the parameter names as keys mapping to parameter values.|
+|minimum_script_version |string |git branch, tag, or version hash |no |The minimum acceptable script version when deciding whether to re-use a past job.|
+|exclude_script_versions|array of strings|git branch, tag, or version hash|no |Script versions to exclude when deciding whether to re-use a past job.|
+|nondeterministic |boolean | |no |If true, never re-use a past job, and flag this job so it will never be considered for re-use.|
+|no_reuse |boolean | |no |If true, do not re-use a past job, but this job may be re-used.|
+
+When a job is executed, the 'script_version' field is resolved to an exact git revision and the git hash for that revision is recorded in 'script_version'. If 'script_version' can't be resolved, the job submission will be rejected.
+
+h3. Reusing jobs
+
+Because Arvados records the exact version of the script, input parameters, and runtime environment [1] that was used to run the job, if the script is deterministic (meaning that the same code version is guaranteed to produce the same outputs from the same inputs) then it is possible to re-use the results of past jobs, and avoid re-running the computation to save time. Arvados uses the following algorithm to determine if a past job can be re-used:
+
+notextile. <div class="spaced-out">
+
+# If 'nondeterministic' or 'no_reuse' are true, always create a new job.
+# Find a list of acceptable values for 'script_version'. If 'minimum_script_version' is specified, this is the set of all revisions in the git commit graph between 'minimum_script_version' and 'script_version' (inclusive) [2]. If 'minimum_script_version' is not specified, only 'script_version' is added to the list. If 'exclude_script_versions' is specified, the listed versions are excluded from the list.
+# Select jobs have the same 'script' and 'script_parameters' attributes, and where the 'script_version' attribute is in the list of acceptable versions. Exclude jobs that failed or set 'nondeterministic' to true.
+# If there is more than one candidate job, check that all selected past jobs actually did produce the same output.
+# If everything passed, re-use one of the selected past jobs (if there is more than one match, which job will be returned is undefined). Otherwise create a new job.
+
+fn1. As of this writing, versioning the runtime environment is still under development.
+
+fn2. This may include parallel branches if there is more than one path between 'minimum_script_version' and 'script_version' in the git commit graph. Use 'exclude_script_versions' to blacklist specific versions.
+
+</div>
+
+h3. Examples
+
+Run the script "crunch_scripts/hash.py" in the repository "you" using the "master" branch head. Arvados is allowed to re-use a previous job if the script_version of the past job is the same as the "master" branch head (i.e., there have not been any subsequent commits to "master").
+
+<notextile><pre>
+{
+ "script": "hash.py",
+ "repository": "<b>you</b>",
+ "script_version": "master",
+ "script_parameters": {
+ "input": "c1bad4b39ca5a924e481008009d94e32+210"
+ }
+}
+</pre></notextile>
+
+Run using exactly the version "d00220fb38d4b85ca8fc28a8151702a2b9d1dec5". Arvados is allowed to re-use a previous job if the "script_version" of that job is also "d00220fb38d4b85ca8fc28a8151702a2b9d1dec5".
+
+<notextile><pre>
+{
+ "script": "hash.py",
+ "repository": "<b>you</b>",
+ "script_version": "d00220fb38d4b85ca8fc28a8151702a2b9d1dec5",
+ "script_parameters": {
+ "input": "c1bad4b39ca5a924e481008009d94e32+210"
+ }
+}
+</pre></notextile>
+
+Arvados is allowed to re-use a previous job if the "script_version" of the past job is between "earlier_version_tag" and the head of the "master" branch (inclusive), but not "blacklisted_version_tag". If there are no previous jobs, run the job using the head of the "master" branch as specified in "script_version".
+
+<notextile><pre>
+{
+ "script": "hash.py",
+ "repository": "<b>you</b>",
+ "minimum_script_version": "earlier_version_tag",
+ "script_version": "master",
+ "exclude_script_versions": ["blacklisted_version_tag"],
+ "script_parameters": {
+ "input": "c1bad4b39ca5a924e481008009d94e32+210"
+ }
+}
+</pre></notextile>
+
+Run the script "crunch_scripts/monte-carlo.py" in the repository "you" using the "master" branch head. Because it is marked as "nondeterministic", never re-use previous jobs, and never re-use this job.
+
+<notextile><pre>
+{
+ "script": "monte-carlo.py",
+ "repository": "<b>you</b>",
+ "script_version": "master",
+ "nondeterministic": true,
+ "script_parameters": {
+ "input": "c1bad4b39ca5a924e481008009d94e32+210"
+ }
+}
+</pre></notextile>
+
+h2. Pipelines
+
+Pipelines consist of a set of "components". Each component is an Arvados job submission, so when a component job is submitted, Arvados may re-use past jobs based on the rules described above.
+
+table(table table-bordered table-condensed).
+|_. Attribute |_. Type |_. Accepted values |_. Required|_. Description|
+|name |string |any |yes |The human-readable name of the pipeline template.|
+|components |object |JSON object containing job submission objects|yes |The component jobs that make up the pipeline, with the component name as the key. |
+
+h3. Script parameters
+
+When used in a pipeline, each parameter in the 'script_parameters' attribute of a component job can specify that the input parameter must be supplied by the user, or the input parameter should be linked to the output of another component. To do this, the value of the parameter should be JSON object containing one of the following attributes:
+
+table(table table-bordered table-condensed).
+|_. Attribute |_. Type |_. Accepted values |_. Description|
+|default |any |any |The default value for this parameter.|
+|required |boolean |true or false |Specifies whether the parameter is required to have a value or not.|
+|dataclass |string |One of 'Collection', 'File' [3], 'number', or 'text' |Data type of this parameter.|
+|output_of |string |the name of another component in the pipeline |Specifies that the value of this parameter should be set to the 'output' attribute of the job that corresponds to the specified component.|
+
+The 'output_of' parameter is especially important, as this is how components are actually linked together to form a pipeline. Component jobs that depend on the output of other components do not run until the parent job completes and has produced output. If the parent job fails, the entire pipeline fails.
+
+fn3. The 'File' type refers to a specific file within a Keep collection in the form 'collection_hash/filename', for example '887cd41e9c613463eab2f0d885c6dd96+83/bob.txt'.
+
+h3. Examples
+
+This is a pipeline named "Filter md5 hash values" with two components, "do_hash" and "filter". The "input" script parameter of the "do_hash" component is required to be filled in by the user, and the expected data type is "Collection". This also specifies that the "input" script parameter of the "filter" component is the output of "do_hash", so "filter" will not run until "do_hash" completes successfully. When the pipeline runs, past jobs that meet the criteria described above may be substituted for either or both components to avoid redundant computation.
+
+<notextile><pre>
+{
+ "name": "Filter md5 hash values",
+ "components": {
+ "do_hash": {
+ "script": "hash.py",
+ "repository": "<b>you</b>",
+ "script_version": "master",
+ "script_parameters": {
+ "input": {
+ "required": true,
+ "dataclass": "Collection"
+ }
+ },
+ },
+ "filter": {
+ "script": "0-filter.py",
+ "repository": "<b>you</b>",
+ "script_version": "master",
+ "script_parameters": {
+ "input": {
+ "output_of": "do_hash"
+ }
+ },
+ }
+ }
+}
+</pre></notextile>
+
+This pipeline consists of three components. The components "thing1" and "thing2" both depend on "cat_in_the_hat". Once the "cat_in_the_hat" job is complete, both "thing1" and "thing2" can run in parallel, because they do not depend on each other.
+
+<notextile><pre>
+{
+ "name": "Wreck the house",
+ "components": {
+ "cat_in_the_hat": {
+ "script": "cat.py",
+ "repository": "<b>you</b>",
+ "script_version": "master",
+ "script_parameters": { }
+ },
+ "thing1": {
+ "script": "thing1.py",
+ "repository": "<b>you</b>",
+ "script_version": "master",
+ "script_parameters": {
+ "input": {
+ "output_of": "cat_in_the_hat"
+ }
+ },
+ },
+ "thing2": {
+ "script": "thing2.py",
+ "repository": "<b>you</b>",
+ "script_version": "master",
+ "script_parameters": {
+ "input": {
+ "output_of": "cat_in_the_hat"
+ }
+ },
+ },
+ }
+}
+</pre></notextile>
+
+This pipeline consists of three components. The component "cleanup" depends on "thing1" and "thing2". Both "thing1" and "thing2" are started immediately and can run in parallel, because they do not depend on each other, but "cleanup" cannot begin until both "thing1" and "thing2" have completed.
+
+<notextile><pre>
+{
+ "name": "Clean the house",
+ "components": {
+ "thing1": {
+ "script": "thing1.py",
+ "repository": "<b>you</b>",
+ "script_version": "master",
+ "script_parameters": { }
+ },
+ "thing2": {
+ "script": "thing2.py",
+ "repository": "<b>you</b>",
+ "script_version": "master",
+ "script_parameters": { }
+ },
+ "cleanup": {
+ "script": "cleanup.py",
+ "repository": "<b>you</b>",
+ "script_version": "master",
+ "script_parameters": {
+ "mess1": {
+ "output_of": "thing1"
+ },
+ "mess2": {
+ "output_of": "thing2"
+ }
+ }
+ }
+ }
+}
+</pre></notextile>
---
layout: default
navsection: userguide
-navmenu: Reference
title: "Command line interface"
...
-h1. Reference: Command Line Interface
-
*First, you should be "logged into an Arvados VM instance":{{site.baseurl}}/user/getting_started/ssh-access.html#login, and have a "working environment.":{{site.baseurl}}/user/getting_started/check-environment.html*
h3. Usage
--- /dev/null
+---
+layout: default
+navsection: userguide
+title: "How Keep works"
+...
+
+In Keep, information is stored in *data blocks*. Data blocks are normally between 1 byte and 64 megabytes in size. If a file exceeds the maximum size of a single data block, the file will be split across multiple data blocks until the entire file can be stored. These data blocks may be stored and replicated across multiple disks, servers, or clusters. Each data block has its own identifier for the contents of that specific data block.
+
+In order to reassemble the file, Keep stores a *collection* data block which lists in sequence the data blocks that make up the original file. A collection data block may store the information for multiple files, including a directory structure.
+
+In this example we will use @c1bad4b39ca5a924e481008009d94e32+210@ which we added to Keep in "the first Keep tutorial":{{ site.baseurl }}/user/tutorials/tutorial-keep.html. First let us examine the contents of this collection using @arv keep get@:
+
+<notextile>
+<pre><code>~$ <span class="userinput">arv keep get c1bad4b39ca5a924e481008009d94e32+210</span>
+. 204e43b8a1185621ca55a94839582e6f+67108864 b9677abbac956bd3e86b1deb28dfac03+67108864 fc15aff2a762b13f521baf042140acec+67108864 323d2a3ce20370c4ca1d3462a344f8fd+25885655 0:227212247:var-GS000016015-ASM.tsv.bz2
+</code></pre>
+</notextile>
+
+The command @arv keep get@ fetches the contents of the collection @c1bad4b39ca5a924e481008009d94e32+210@. In this example, this collection includes a single file @var-GS000016015-ASM.tsv.bz2@ which is 227212247 bytes long, and is stored using four sequential data blocks, @204e43b8a1185621ca55a94839582e6f+67108864@, @b9677abbac956bd3e86b1deb28dfac03+67108864@, @fc15aff2a762b13f521baf042140acec+67108864@, and @323d2a3ce20370c4ca1d3462a344f8fd+25885655@.
+
+Let's use @arv keep get@ to download the first data block:
+
+notextile. <pre><code>~$ <span class="userinput">cd /scratch/<b>you</b></span>
+/scratch/<b>you</b>$ <span class="userinput">arv keep get 204e43b8a1185621ca55a94839582e6f+67108864 > block1</span></code></pre>
+
+{% include 'notebox_begin' %}
+
+When you run this command, you may get this API warning:
+
+notextile. <pre><code>WARNING:root:API lookup failed for collection 204e43b8a1185621ca55a94839582e6f+67108864 (<class 'apiclient.errors.HttpError'>: <HttpError 404 when requesting https://qr1hi.arvadosapi.com/arvados/v1/collections/204e43b8a1185621ca55a94839582e6f%2B67108864?alt=json returned "Not Found">)</code></pre>
+
+This happens because @arv keep get@ tries to find a collection with this identifier. When that fails, it emits this warning, then looks for a datablock instead, which succeeds.
+
+{% include 'notebox_end' %}
+
+Let's look at the size and compute the md5 hash of @block1@:
+
+<notextile>
+<pre><code>/scratch/<b>you</b>$ <span class="userinput">ls -l block1</span>
+-rw-r--r-- 1 you group 67108864 Dec 9 20:14 block1
+/scratch/<b>you</b>$ <span class="userinput">md5sum block1</span>
+204e43b8a1185621ca55a94839582e6f block1
+</code></pre>
+</notextile>
+
+Notice that the block identifer <code>204e43b8a1185621ca55a94839582e6f+67108864</code> consists of:
+* the md5 hash of @block1@, @204e43b8a1185621ca55a94839582e6f@, plus
+* the size of @block1@, @67108864@.
--- /dev/null
+---
+layout: default
+navsection: userguide
+title: "Running a pipeline on the command line"
+...
+
+In "Writing a pipeline":{{ site.baseurl }}/user/tutorials/tutorial-firstscript.html, we learned how to create a pipeline template on the command-line. Let's create one that doesn't require any user input to start:
+
+<notextile>
+<pre><code>~$ <span class="userinput">cat >the_pipeline <<EOF
+{
+ "name":"Filter md5 hash values",
+ "components":{
+ "do_hash":{
+ "script":"hash.py",
+ "script_parameters":{
+ "input": "887cd41e9c613463eab2f0d885c6dd96+83"
+ },
+ "repository":"$USER",
+ "script_version":"master"
+ },
+ "filter":{
+ "script":"0-filter.py",
+ "script_parameters":{
+ "input":{
+ "output_of":"do_hash"
+ }
+ },
+ "repository":"$USER",
+ "script_version":"master"
+ }
+ }
+}
+EOF</span>
+~$ <span class="userinput">arv pipeline_template create --pipeline-template "$(cat the_pipeline)"</span></code></pre>
+</notextile>
+
+(Your shell should automatically fill in @$USER@ with your login name. The JSON that gets saved should have @"repository"@ pointed at your personal git repository.)
+
+You can run this pipeline from the command line using @arv pipeline run@, filling in the UUID that you received from @arv pipeline_template create@:
+
+<notextile>
+<pre><code>~$ <span class="userinput">arv pipeline run --run-here --template qr1hi-p5p6p-xxxxxxxxxxxxxxx</span>
+2013-12-16 14:08:40 +0000 -- pipeline_instance qr1hi-d1hrv-vxzkp38nlde9yyr
+do_hash qr1hi-8i9sb-hoyc2u964ecv1s6 queued 2013-12-16T14:08:40Z
+filter - -
+
+2013-12-16 14:08:51 +0000 -- pipeline_instance qr1hi-d1hrv-vxzkp38nlde9yyr
+do_hash qr1hi-8i9sb-hoyc2u964ecv1s6 1ed9ed18ef31ad21bcabcfeff7777bae+162
+filter qr1hi-8i9sb-w5k40fztqgg9i2x queued 2013-12-16T14:08:50Z
+
+2013-12-16 14:09:01 +0000 -- pipeline_instance qr1hi-d1hrv-vxzkp38nlde9yyr
+do_hash qr1hi-8i9sb-hoyc2u964ecv1s6 1ed9ed18ef31ad21bcabcfeff7777bae+162
+filter qr1hi-8i9sb-w5k40fztqgg9i2x d3bcc2ee0f0ea31049000c721c0f3a2a+56
+</code></pre>
+</notextile>
+
+This instantiates your pipeline and displays a live feed of its status. The new pipeline instance will also show up on Workbench *Activity* %(rarr)→% *Recent pipeline instances* page.
+
+Arvados adds each pipeline component to the job queue as its dependencies are satisfied (or immediately if it has no dependencies) and finishes when all components are completed or failed and there is no more work left to do.
+
+The Keep locators of the output of each of @"do_hash"@ and @"filter"@ component are available from the output log shown above. The output is also available on the Workbench by navigating to *Activity* %(rarr)→% *Recent pipeline instances* %(rarr)→% pipeline UUID under the *Instance* column %(rarr)→% *output* column.
+
+<notextile>
+<pre><code>~$ <span class="userinput">arv keep get 1ed9ed18ef31ad21bcabcfeff7777bae+162/md5sum.txt</span>
+0f1d6bcf55c34bed7f92a805d2d89bbf 887cd41e9c613463eab2f0d885c6dd96+83/./alice.txt
+504938460ef369cd275e4ef58994cffe 887cd41e9c613463eab2f0d885c6dd96+83/./bob.txt
+8f3b36aff310e06f3c5b9e95678ff77a 887cd41e9c613463eab2f0d885c6dd96+83/./carol.txt
+~$ <span class="userinput">arv keep get d3bcc2ee0f0ea31049000c721c0f3a2a+56/0-filter.txt</span>
+0f1d6bcf55c34bed7f92a805d2d89bbf 887cd41e9c613463eab2f0d885c6dd96+83/./alice.txt
+</code></pre>
+</notextile>
+
+Indeed, the filter has picked out just the "alice" file as having a hash that starts with 0.
+
+h3. Running a pipeline with different parameters
+
+Notice that the pipeline template explicitly specifies the Keep locator for the input:
+
+<notextile>
+<pre><code>...
+ "do_hash":{
+ "script_parameters":{
+ "input": "887cd41e9c613463eab2f0d885c6dd96+83"
+ },
+ }
+...
+</code></pre>
+</notextile>
+
+You can specify values for pipeline component script_parameters like this:
+
+<notextile>
+<pre><code>~$ <span class="userinput">arv pipeline run --run-here --template qr1hi-p5p6p-xxxxxxxxxxxxxxx do_hash::input=c1bad4b39ca5a924e481008009d94e32+210</span>
+2013-12-17 20:31:24 +0000 -- pipeline_instance qr1hi-d1hrv-tlkq20687akys8e
+do_hash qr1hi-8i9sb-rffhuay4jryl2n2 queued 2013-12-17T20:31:24Z
+filter - -
+
+2013-12-17 20:31:34 +0000 -- pipeline_instance qr1hi-d1hrv-tlkq20687akys8e
+do_hash qr1hi-8i9sb-rffhuay4jryl2n2 {:done=>1, :running=>1, :failed=>0, :todo=>0}
+filter - -
+
+2013-12-17 20:31:55 +0000 -- pipeline_instance qr1hi-d1hrv-tlkq20687akys8e
+do_hash qr1hi-8i9sb-rffhuay4jryl2n2 50cafdb29cc21dd6eaec85ba9e0c6134+56
+filter qr1hi-8i9sb-j347g1sqovdh0op queued 2013-12-17T20:31:55Z
+
+2013-12-17 20:32:05 +0000 -- pipeline_instance qr1hi-d1hrv-tlkq20687akys8e
+do_hash qr1hi-8i9sb-rffhuay4jryl2n2 50cafdb29cc21dd6eaec85ba9e0c6134+56
+filter qr1hi-8i9sb-j347g1sqovdh0op 490cd451c8108824b8a17e3723e1f236+19
+</code></pre>
+</notextile>
+
+Now check the output:
+
+<notextile>
+<pre><code>~$ <span class="userinput">arv keep get 50cafdb29cc21dd6eaec85ba9e0c6134+56/md5sum.txt</span>
+44b8ae3fde7a8a88d2f7ebd237625b4f c1bad4b39ca5a924e481008009d94e32+210/./var-GS000016015-ASM.tsv.bz2
+~$ <span class="userinput">arv keep get 490cd451c8108824b8a17e3723e1f236+19/0-filter.txt</span>
+</code></pre>
+</notextile>
+
+Since none of the files in the collection have hash code that start with 0, the output of the filter component is empty.
---
layout: default
navsection: userguide
-navmenu: Tutorials
title: "Using GATK with Arvados"
-
...
-h1. Using GATK with Arvados
-
This tutorial demonstrates how to use the Genome Analysis Toolkit (GATK) with Arvados. In this example we will install GATK and then create a VariantFiltration job to assign pass/fail scores to variants in a VCF file.
*This tutorial assumes that you are "logged into an Arvados VM instance":{{site.baseurl}}/user/getting_started/ssh-access.html#login, and have a "working environment.":{{site.baseurl}}/user/getting_started/check-environment.html*
h2. Installing GATK
-Download the GATK binary tarball[1] -- e.g., @GenomeAnalysisTK-2.6-4.tar.bz2@ -- and "copy it to your Arvados VM":tutorial-keep.html.
+Download the GATK binary tarball[1] -- e.g., @GenomeAnalysisTK-2.6-4.tar.bz2@ -- and "copy it to your Arvados VM":{{site.baseurl}}/user/tutorials/tutorial-keep.html.
<notextile>
<pre><code>~$ <span class="userinput">arv keep put GenomeAnalysisTK-2.6-4.tar.bz2</span>
~$ <span class="userinput">cat >the_job <<EOF
{
"script":"GATK2-VariantFiltration",
+ "repository":"arvados",
"script_version":"$src_version",
"script_parameters":
{
---
layout: default
navsection: userguide
-navmenu: Tutorials
title: "Debugging a Crunch script"
-
...
-h1. Debugging a Crunch script
-
To test changes to a script by running a job, the change must be pushed into @git@, the job queued asynchronously, and the actual execution may be run on any compute server. As a result, debugging a script can be difficult and time consuming. This tutorial demonstrates using @arv-crunch-job@ to run your job in your local VM. This avoids the job queue and allows you to execute the script from your uncomitted git tree.
*This tutorial assumes that you are "logged into an Arvados VM instance":{{site.baseurl}}/user/getting_started/ssh-access.html#login, and have a "working environment.":{{site.baseurl}}/user/getting_started/check-environment.html*
-This tutorial uses _you_ to denote your username. Replace _you_ with your user name in all the following examples.
+This tutorial uses *@you@* to denote your username. Replace *@you@* with your user name in all the following examples.
h2. Create a new script
-Change to your git directory and create a new script in "crunch_scripts/".
+Change to your git directory and create a new script in @crunch_scripts/@.
<notextile>
<pre><code>~$ <span class="userinput">cd <b>you</b>/crunch_scripts</span>
h2. Using arv-crunch-job to run the job in your VM
-Instead of a git commit hash, we provide the path to the directory in the "script_version" parameter. The script specified in "script" will actually be searched for in the "crunch_scripts/" subdirectory of the directory specified "script_version". Although we are running the script locally, the script still requires access to the Arvados API server and Keep storage service. The job will be recorded in the Arvados job history, and visible in Workbench.
+Instead of a git commit hash, we provide the path to the directory in the "script_version" parameter. The script specified in "script" will actually be searched for in the @crunch_scripts/@ subdirectory of the directory specified "script_version". Although we are running the script locally, the script still requires access to the Arvados API server and Keep storage service. The job will be recorded in the Arvados job history, and visible in Workbench.
<notextile>
<pre><code>~/<b>you</b>/crunch_scripts$ <span class="userinput">cat >~/the_job <<EOF
{
+ "repository":"",
"script":"hello-world.py",
- "script_version":"/home/you/you",
+ "script_version":"$HOME/$USER",
"script_parameters":{}
}
EOF</span>
-~/<b>you</b>/crunch_scripts</span>$ <span class="userinput">arv-crunch-job --job "$(cat ~/the_job)"</span>
+</code></pre>
+</notextile>
+
+Your shell should fill in values for @$HOME@ and @$USER@ so that the saved JSON points "script_version" at the directory with your checkout. Now you can run that job:
+
+<notextile>
+<pre><code>~/<b>you</b>/crunch_scripts</span>$ <span class="userinput">arv-crunch-job --job "$(cat ~/the_job)"</span>
2013-12-12_21:36:42 qr1hi-8i9sb-okzukfzkpbrnhst 29827 check slurm allocation
2013-12-12_21:36:42 qr1hi-8i9sb-okzukfzkpbrnhst 29827 node localhost - 1 slots
2013-12-12_21:36:42 qr1hi-8i9sb-okzukfzkpbrnhst 29827 start
2013-12-12_21:36:42 qr1hi-8i9sb-okzukfzkpbrnhst 29827 script hello-world.py
-2013-12-12_21:36:42 qr1hi-8i9sb-okzukfzkpbrnhst 29827 script_version /home/you/you
+2013-12-12_21:36:42 qr1hi-8i9sb-okzukfzkpbrnhst 29827 script_version /home/<b>you</b>/<b>you</b>
2013-12-12_21:36:42 qr1hi-8i9sb-okzukfzkpbrnhst 29827 script_parameters {}
2013-12-12_21:36:42 qr1hi-8i9sb-okzukfzkpbrnhst 29827 runtime_constraints {"max_tasks_per_node":0}
2013-12-12_21:36:42 qr1hi-8i9sb-okzukfzkpbrnhst 29827 start level 0
2013-12-12_21:36:42 qr1hi-8i9sb-okzukfzkpbrnhst 29827 0 stderr hello world
2013-12-12_21:36:43 qr1hi-8i9sb-okzukfzkpbrnhst 29827 0 child 29834 on localhost.1 exit 0 signal 0 success=
2013-12-12_21:36:43 qr1hi-8i9sb-okzukfzkpbrnhst 29827 0 failure (#1, permanent) after 0 seconds
-2013-12-12_21:36:43 qr1hi-8i9sb-okzukfzkpbrnhst 29827 0 output
+2013-12-12_21:36:43 qr1hi-8i9sb-okzukfzkpbrnhst 29827 0 output
2013-12-12_21:36:43 qr1hi-8i9sb-okzukfzkpbrnhst 29827 Every node has failed -- giving up on this round
2013-12-12_21:36:43 qr1hi-8i9sb-okzukfzkpbrnhst 29827 wait for last 0 children to finish
2013-12-12_21:36:43 qr1hi-8i9sb-okzukfzkpbrnhst 29827 status: 0 done, 0 running, 0 todo
The script's output is captured in the log, which is useful for print statement debugging. However, although this script returned a status code of 0 (success), the job failed. Why? For a job to complete successfully scripts must explicitly add their output to Keep, and then tell Arvados about it. Here is a second try:
<notextile>
-<pre><code>~/<b>you</b>/crunch_scripts$ <span class="userinput">cat >hello-world.py <<EOF
+<pre><code>~/<b>you</b>/crunch_scripts$ <span class="userinput">cat >hello-world-fixed.py <<EOF
#!/usr/bin/env python
import arvados
~/<b>you</b>/crunch_scripts$ <span class="userinput">chmod +x hello-world-fixed.py</span>
~/<b>you</b>/crunch_scripts$ <span class="userinput">cat >~/the_job <<EOF
{
+ "repository":"",
"script":"hello-world-fixed.py",
- "script_version":"/home/you/you",
+ "script_version":"$HOME/$USER",
"script_parameters":{}
}
EOF</span>
2013-12-12_21:56:59 qr1hi-8i9sb-79260ykfew5trzl 31578 check slurm allocation
2013-12-12_21:56:59 qr1hi-8i9sb-79260ykfew5trzl 31578 node localhost - 1 slots
2013-12-12_21:57:00 qr1hi-8i9sb-79260ykfew5trzl 31578 start
-2013-12-12_21:57:00 qr1hi-8i9sb-79260ykfew5trzl 31578 script hello-world.py
-2013-12-12_21:57:00 qr1hi-8i9sb-79260ykfew5trzl 31578 script_version /home/you/you
+2013-12-12_21:57:00 qr1hi-8i9sb-79260ykfew5trzl 31578 script hello-world-fixed.py
+2013-12-12_21:57:00 qr1hi-8i9sb-79260ykfew5trzl 31578 script_version /home/<b>you</b>/<b>you</b>
2013-12-12_21:57:00 qr1hi-8i9sb-79260ykfew5trzl 31578 script_parameters {}
2013-12-12_21:57:00 qr1hi-8i9sb-79260ykfew5trzl 31578 runtime_constraints {"max_tasks_per_node":0}
2013-12-12_21:57:00 qr1hi-8i9sb-79260ykfew5trzl 31578 start level 0
2013-12-12_21:57:02 qr1hi-8i9sb-79260ykfew5trzl 31578 Freeze not implemented
2013-12-12_21:57:02 qr1hi-8i9sb-79260ykfew5trzl 31578 collate
2013-12-12_21:57:02 qr1hi-8i9sb-79260ykfew5trzl 31578 output 576c44d762ba241b0a674aa43152b52a+53
+WARNING:root:API lookup failed for collection 576c44d762ba241b0a674aa43152b52a+53 (<class 'apiclient.errors.HttpError'>: <HttpError 404 when requesting https://qr1hi.arvadosapi.com/arvados/v1/collections/576c44d762ba241b0a674aa43152b52a%2B53?alt=json returned "Not Found">)
2013-12-12_21:57:03 qr1hi-8i9sb-79260ykfew5trzl 31578 finish
-2013-12-12_21:57:04 qr1hi-8i9sb-79260ykfew5trzl 31578 meta key is 9f937693334d0c9234ccc1f808ee7117+1761
</code></pre>
</notextile>
+(The WARNING issued near the end of the script may be safely ignored here; it is the Arvados SDK letting you know that it could not find a collection named @576c44d762ba241b0a674aa43152b52a+53@ and that it is going to try looking up a block by that name instead.)
+
The job succeeded, with output in Keep object @576c44d762ba241b0a674aa43152b52a+53@. Let's look at our output:
<notextile>
notextile. <pre><code>~$ <span class="userinput">export KEEP_LOCAL_STORE=/tmp</span></code></pre>
-Next, "parallel tasks.":tutorial-parallel.html
--- /dev/null
+---
+layout: default
+navsection: userguide
+title: "Running a Crunch job on the command line"
+...
+
+This tutorial introduces how to run individual Crunch jobs using the @arv@ command line tool.
+
+*This tutorial assumes that you are "logged into an Arvados VM instance":{{site.baseurl}}/user/getting_started/ssh-access.html#login, and have a "working environment.":{{site.baseurl}}/user/getting_started/check-environment.html*
+
+You will create a job to run the "hash" Crunch script. The "hash" script computes the md5 hash of each file in a collection.
+
+h2. Jobs
+
+Crunch pipelines consist of one or more jobs. A "job" is a single run of a specific version of a Crunch script with a specific input. You can also run jobs individually.
+
+A request to run a Crunch job are is described using a JSON object. For example:
+
+<notextile>
+<pre><code>~$ <span class="userinput">cat >~/the_job <<EOF
+{
+ "script": "hash",
+ "repository": "arvados",
+ "script_version": "master",
+ "script_parameters": {
+ "input": "c1bad4b39ca5a924e481008009d94e32+210"
+ },
+ "no_reuse": "true"
+}
+EOF
+</code></pre>
+</notextile>
+
+* @cat@ is a standard Unix utility that writes a sequence of input to standard output.
+* @<<EOF@ tells the shell to direct the following lines into the standard input for @cat@ up until it sees the line @EOF@.
+* @>~/the_job@ redirects standard output to a file called @~/the_job@.
+* @"repository"@ is the name of a git repository to search for the script version. You can access a list of available git repositories on the Arvados Workbench under "*Compute* %(rarr)→% *Code repositories*":https://{{site.arvados_workbench_host}}/repositories.
+* @"script_version"@ specifies the version of the script that you wish to run. This can be in the form of an explicit git revision hash, a tag, or a branch (in which case it will use the most recent commit on the specified branch). Arvados logs the script version that was used in the run, enabling you to go back and re-run any past job with the guarantee that the exact same code will be used as was used in the previous run.
+* @"script"@ specifies the name of the script to run. The script is searched for in the @crunch_scripts/@ subdirectory of the git repository.
+* @"script_parameters"@ are provided to the script. In this case, the input is the PGP data Collection that we "put in Keep earlier":{{site.baseurl}}/user/tutorials/tutorial-keep.html.
+* Setting the @"no_reuse"@ flag tells Crunch not to reuse work from past jobs. This helps ensure that you can watch a new Job process for the rest of this tutorial, without reusing output from a past run that you made, or somebody else marked as public. (If you want to experiment, after the first run below finishes, feel free to edit this job to remove the @"no_reuse"@ line and resubmit it. See what happens!)
+
+Use @arv job create@ to actually submit the job. It should print out a JSON object which describes the newly created job:
+
+<notextile>
+<pre><code>~$ <span class="userinput">arv job create --job "$(cat ~/the_job)"</span>
+{
+ "href":"https://qr1hi.arvadosapi.com/arvados/v1/jobs/qr1hi-8i9sb-1pm1t02dezhupss",
+ "kind":"arvados#job",
+ "etag":"ax3cn7w9whq2hdh983yxvq09p",
+ "uuid":"qr1hi-8i9sb-1pm1t02dezhupss",
+ "owner_uuid":"qr1hi-tpzed-9zdpkpni2yddge6",
+ "created_at":"2013-12-16T20:44:32Z",
+ "modified_by_client_uuid":"qr1hi-ozdt8-obw7foaks3qjyej",
+ "modified_by_user_uuid":"qr1hi-tpzed-9zdpkpni2yddge6",
+ "modified_at":"2013-12-16T20:44:32Z",
+ "updated_at":"2013-12-16T20:44:33Z",
+ "submit_id":null,
+ "priority":null,
+ "script":"hash",
+ "script_parameters":{
+ "input":"c1bad4b39ca5a924e481008009d94e32+210"
+ },
+ "script_version":"d9cd657b733d578ac0d2167dd75967aa4f22e0ac",
+ "cancelled_at":null,
+ "cancelled_by_client_uuid":null,
+ "cancelled_by_user_uuid":null,
+ "started_at":null,
+ "finished_at":null,
+ "output":null,
+ "success":null,
+ "running":null,
+ "is_locked_by_uuid":null,
+ "log":null,
+ "runtime_constraints":{},
+ "tasks_summary":{},
+ "dependencies":[
+ "c1bad4b39ca5a924e481008009d94e32+210"
+ ],
+ "log_stream_href":"https://qr1hi.arvadosapi.com/arvados/v1/jobs/qr1hi-8i9sb-1pm1t02dezhupss/log_tail_follow"
+}
+</code></pre>
+</notextile>
+
+The job is now queued and will start running as soon as it reaches the front of the queue. Fields to pay attention to include:
+
+ * @"uuid"@ is the unique identifier for this specific job.
+ * @"script_version"@ is the actual revision of the script used. This is useful if the version was described using the "repository:branch" format.
+
+h2. Monitor job progress
+
+Go to the "Workbench dashboard":https://{{site.arvados_workbench_host}} and visit *Activity* %(rarr)→% *Recent jobs*. Your job should be near the top of the table. This table refreshes automatically. When the job has completed successfully, it will show <span class="label label-success">finished</span> in the *Status* column.
+
+On the command line, you can access log messages while the job runs using @arv job log_tail_follow@:
+
+notextile. <pre><code>~$ <span class="userinput">arv job log_tail_follow --uuid qr1hi-8i9sb-xxxxxxxxxxxxxxx</span></code></pre>
+
+This will print out the last several lines of the log for that job.
+
+h2. Inspect the job output
+
+On the "Workbench dashboard":https://{{site.arvados_workbench_host}}, look for the *Output* column of the *Recent jobs* table. Click on the link under *Output* for your job to go to the files page with the job output. The files page lists all the files that were output by the job. Click on the link under the *file* column to view a file, or click on the download icon <span class="glyphicon glyphicon-download-alt"></span> to download the output file.
+
+On the command line, you can use @arv job get@ to access a JSON object describing the output:
+
+<notextile>
+<pre><code>~$ <span class="userinput">arv job get --uuid qr1hi-8i9sb-xxxxxxxxxxxxxxx</span>
+{
+ "href":"https://qr1hi.arvadosapi.com/arvados/v1/jobs/qr1hi-8i9sb-1pm1t02dezhupss",
+ "kind":"arvados#job",
+ "etag":"1bk98tdj0qipjy0rvrj03ta5r",
+ "uuid":"qr1hi-8i9sb-1pm1t02dezhupss",
+ "owner_uuid":"qr1hi-tpzed-9zdpkpni2yddge6",
+ "created_at":"2013-12-16T20:44:32Z",
+ "modified_by_client_uuid":null,
+ "modified_by_user_uuid":"qr1hi-tpzed-9zdpkpni2yddge6",
+ "modified_at":"2013-12-16T20:44:55Z",
+ "updated_at":"2013-12-16T20:44:55Z",
+ "submit_id":null,
+ "priority":null,
+ "script":"hash",
+ "script_parameters":{
+ "input":"c1bad4b39ca5a924e481008009d94e32+210"
+ },
+ "script_version":"d9cd657b733d578ac0d2167dd75967aa4f22e0ac",
+ "cancelled_at":null,
+ "cancelled_by_client_uuid":null,
+ "cancelled_by_user_uuid":null,
+ "started_at":"2013-12-16T20:44:36Z",
+ "finished_at":"2013-12-16T20:44:53Z",
+ "output":"dd755dbc8d49a67f4fe7dc843e4f10a6+54",
+ "success":true,
+ "running":false,
+ "is_locked_by_uuid":"qr1hi-tpzed-9zdpkpni2yddge6",
+ "log":"2afdc6c8b67372ffd22d8ce89d35411f+91",
+ "runtime_constraints":{},
+ "tasks_summary":{
+ "done":2,
+ "running":0,
+ "failed":0,
+ "todo":0
+ },
+ "dependencies":[
+ "c1bad4b39ca5a924e481008009d94e32+210"
+ ],
+ "log_stream_href":null
+}
+</code></pre>
+</notextile>
+
+* @"output"@ is the unique identifier for this specific job's output. This is a Keep collection. Because the output of Arvados jobs should be deterministic, the known expected output is <code>dd755dbc8d49a67f4fe7dc843e4f10a6+54</code>.
+
+Now you can list the files in the collection:
+
+<notextile>
+<pre><code>~$ <span class="userinput">arv keep ls dd755dbc8d49a67f4fe7dc843e4f10a6+54</span>
+./md5sum.txt
+</code></pre>
+</notextile>
+
+This collection consists of the @md5sum.txt@ file. Use @arv keep get@ to show the contents of the @md5sum.txt@ file:
+
+<notextile>
+<pre><code>~$ <span class="userinput">arv keep get dd755dbc8d49a67f4fe7dc843e4f10a6+54/md5sum.txt</span>
+44b8ae3fde7a8a88d2f7ebd237625b4f ./var-GS000016015-ASM.tsv.bz2
+</code></pre>
+</notextile>
+
+This md5 hash matches the md5 hash which we "computed earlier":{{site.baseurl}}/user/tutorials/tutorial-keep.html.
+
+h2. The job log
+
+When the job completes, you can access the job log. On the Workbench, visit *Activity* %(rarr)→% *Recent jobs* %(rarr)→% your job's UUID under the *uuid* column %(rarr)→% the collection link on the *log* row.
+
+On the command line, the Keep identifier listed in the @"log"@ field from @arv job get@ specifies a collection. You can list the files in the collection:
+
+<notextile>
+<pre><code>~$ <span class="userinput">arv keep ls xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx+91</span>
+./qr1hi-8i9sb-xxxxxxxxxxxxxxx.log.txt
+</code></pre>
+</notextile>
+
+The log collection consists of one log file named with the job's UUID. You can access it using @arv keep get@:
+
+<notextile>
+<pre><code>~$ <span class="userinput">arv keep get xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx+91/qr1hi-8i9sb-xxxxxxxxxxxxxxx.log.txt</span>
+2013-12-16_20:44:35 qr1hi-8i9sb-xxxxxxxxxxxxxxx 7575 check slurm allocation
+2013-12-16_20:44:35 qr1hi-8i9sb-xxxxxxxxxxxxxxx 7575 node compute13 - 8 slots
+2013-12-16_20:44:36 qr1hi-8i9sb-xxxxxxxxxxxxxxx 7575 start
+2013-12-16_20:44:36 qr1hi-8i9sb-xxxxxxxxxxxxxxx 7575 Install revision d9cd657b733d578ac0d2167dd75967aa4f22e0ac
+2013-12-16_20:44:37 qr1hi-8i9sb-xxxxxxxxxxxxxxx 7575 Clean-work-dir exited 0
+2013-12-16_20:44:37 qr1hi-8i9sb-xxxxxxxxxxxxxxx 7575 Install exited 0
+2013-12-16_20:44:37 qr1hi-8i9sb-xxxxxxxxxxxxxxx 7575 script hash
+2013-12-16_20:44:37 qr1hi-8i9sb-xxxxxxxxxxxxxxx 7575 script_version d9cd657b733d578ac0d2167dd75967aa4f22e0ac
+2013-12-16_20:44:37 qr1hi-8i9sb-xxxxxxxxxxxxxxx 7575 script_parameters {"input":"c1bad4b39ca5a924e481008009d94e32+210"}
+2013-12-16_20:44:37 qr1hi-8i9sb-xxxxxxxxxxxxxxx 7575 runtime_constraints {"max_tasks_per_node":0}
+2013-12-16_20:44:37 qr1hi-8i9sb-xxxxxxxxxxxxxxx 7575 start level 0
+2013-12-16_20:44:37 qr1hi-8i9sb-xxxxxxxxxxxxxxx 7575 status: 0 done, 0 running, 1 todo
+2013-12-16_20:44:38 qr1hi-8i9sb-xxxxxxxxxxxxxxx 7575 0 job_task qr1hi-ot0gb-23c1k3kwrf8da62
+2013-12-16_20:44:38 qr1hi-8i9sb-xxxxxxxxxxxxxxx 7575 0 child 7681 started on compute13.1
+2013-12-16_20:44:38 qr1hi-8i9sb-xxxxxxxxxxxxxxx 7575 status: 0 done, 1 running, 0 todo
+2013-12-16_20:44:39 qr1hi-8i9sb-xxxxxxxxxxxxxxx 7575 0 child 7681 on compute13.1 exit 0 signal 0 success=true
+2013-12-16_20:44:39 qr1hi-8i9sb-xxxxxxxxxxxxxxx 7575 0 success in 1 seconds
+2013-12-16_20:44:39 qr1hi-8i9sb-xxxxxxxxxxxxxxx 7575 0 output
+2013-12-16_20:44:39 qr1hi-8i9sb-xxxxxxxxxxxxxxx 7575 wait for last 0 children to finish
+2013-12-16_20:44:39 qr1hi-8i9sb-xxxxxxxxxxxxxxx 7575 status: 1 done, 0 running, 1 todo
+2013-12-16_20:44:39 qr1hi-8i9sb-xxxxxxxxxxxxxxx 7575 start level 1
+2013-12-16_20:44:39 qr1hi-8i9sb-xxxxxxxxxxxxxxx 7575 status: 1 done, 0 running, 1 todo
+2013-12-16_20:44:39 qr1hi-8i9sb-xxxxxxxxxxxxxxx 7575 1 job_task qr1hi-ot0gb-iwr0o3unqothg28
+2013-12-16_20:44:39 qr1hi-8i9sb-xxxxxxxxxxxxxxx 7575 1 child 7716 started on compute13.1
+2013-12-16_20:44:39 qr1hi-8i9sb-xxxxxxxxxxxxxxx 7575 status: 1 done, 1 running, 0 todo
+2013-12-16_20:44:52 qr1hi-8i9sb-xxxxxxxxxxxxxxx 7575 1 child 7716 on compute13.1 exit 0 signal 0 success=true
+2013-12-16_20:44:52 qr1hi-8i9sb-xxxxxxxxxxxxxxx 7575 1 success in 13 seconds
+2013-12-16_20:44:52 qr1hi-8i9sb-xxxxxxxxxxxxxxx 7575 1 output dd755dbc8d49a67f4fe7dc843e4f10a6+54
+2013-12-16_20:44:52 qr1hi-8i9sb-xxxxxxxxxxxxxxx 7575 wait for last 0 children to finish
+2013-12-16_20:44:52 qr1hi-8i9sb-xxxxxxxxxxxxxxx 7575 status: 2 done, 0 running, 0 todo
+2013-12-16_20:44:52 qr1hi-8i9sb-xxxxxxxxxxxxxxx 7575 release job allocation
+2013-12-16_20:44:52 qr1hi-8i9sb-xxxxxxxxxxxxxxx 7575 Freeze not implemented
+2013-12-16_20:44:52 qr1hi-8i9sb-xxxxxxxxxxxxxxx 7575 collate
+2013-12-16_20:44:53 qr1hi-8i9sb-xxxxxxxxxxxxxxx 7575 output dd755dbc8d49a67f4fe7dc843e4f10a6+54+K@qr1hi
+2013-12-16_20:44:53 qr1hi-8i9sb-xxxxxxxxxxxxxxx 7575 finish
+</code></pre>
+</notextile>
---
layout: default
navsection: userguide
-navmenu: Tutorials
title: "Parallel Crunch tasks"
-
...
-h1. Parallel Crunch tasks
-
-In the tutorial "writing a crunch script,":tutorial-firstscript.html our script used a "for" loop to compute the md5 hashes for each file in sequence. This approach, while simple, is not able to take advantage of the compute cluster with multiple nodes and cores to speed up computation by running tasks in parallel. This tutorial will demonstrate how to create parallel Crunch tasks.
+In the previous tutorials, we used @arvados.job_setup.one_task_per_input_file()@ to automatically parallelize our jobs by creating a separate task per file. For some types of jobs, you may need to split the work up differently, for example creating tasks to process different segments of a single large file. In this this tutorial will demonstrate how to create Crunch tasks directly.
Start by entering the @crunch_scripts@ directory of your git repository:
notextile. <pre>~/<b>you</b>/crunch_scripts$ <code class="userinput">nano parallel-hash.py</code></pre>
-Add the following code to compute the md5 hash of each file in a
+Add the following code to compute the md5 hash of each file in a collection:
-<pre><code class="userinput">{% include 'parallel_hash_script_py' %}</code></pre>
+<notextile> {% code 'parallel_hash_script_py' as python %} </notextile>
Make the file executable:
</code></pre>
</notextile>
-You should now be able to run your new script using Crunch, with "script" referring to our new "parallel-hash.py" script. We will use a different input from our previous examples. We will use @887cd41e9c613463eab2f0d885c6dd96+83@ which consists of three files, "alice.txt", "bob.txt" and "carol.txt" (the example collection used previously in "fetching data from Arvados using Keep":tutorial-keep.html).
+You should now be able to run your new script using Crunch, with "script" referring to our new "parallel-hash.py" script. We will use a different input from our previous examples. We will use @887cd41e9c613463eab2f0d885c6dd96+83@ which consists of three files, "alice.txt", "bob.txt" and "carol.txt" (the example collection used previously in "fetching data from Arvados using Keep":{{site.baseurl}}/user/tutorials/tutorial-keep.html#dir).
<notextile>
<pre><code>~/<b>you</b>/crunch_scripts$ <span class="userinput">cat >~/the_job <<EOF
{
"script": "parallel-hash.py",
- "script_version": "you:master",
+ "repository": "$USER",
+ "script_version": "master",
"script_parameters":
{
"input": "887cd41e9c613463eab2f0d885c6dd96+83"
</code></pre>
</notextile>
+(Your shell should automatically fill in @$USER@ with your login name. The job JSON that gets saved should have @"repository"@ pointed at your personal git repository.)
+
Because the job ran in parallel, each instance of parallel-hash creates a separate @md5sum.txt@ as output. Arvados automatically collates theses files into a single collection, which is the output of the job:
<notextile>
-<pre><code>~/<b>you</b>/crunch_scripts$ <span class="userinput">arv keep get e2ccd204bca37c77c0ba59fc470cd0f7+162</span>
-md5sum.txt
-md5sum.txt
-md5sum.txt
+<pre><code>~/<b>you</b>/crunch_scripts$ <span class="userinput">arv keep ls e2ccd204bca37c77c0ba59fc470cd0f7+162</span>
+./md5sum.txt
~/<b>you</b>/crunch_scripts$ <span class="userinput">arv keep get e2ccd204bca37c77c0ba59fc470cd0f7+162/md5sum.txt</span>
0f1d6bcf55c34bed7f92a805d2d89bbf alice.txt
504938460ef369cd275e4ef58994cffe bob.txt
</code></pre>
</notextile>
-h2. The one job per file pattern
-
-This example demonstrates how to schedule a new task per file. Because this is a common pattern, the Crunch Python API contains a convenience function to "queue a task for each input file":{{site.baseurl}}/sdk/python/crunch-utility-libraries.html#one_task_per_input which reduces the amount of boilerplate code required to handle parallel jobs.
-
-Next, "Constructing a Crunch pipeline":tutorial-new-pipeline.html
---
layout: default
navsection: userguide
-navmenu: Tutorials
title: "Querying the Metadata Database"
-
...
-h1. Querying the Metadata Database
-
This tutorial introduces the Arvados Metadata Database. The Metadata Database stores information about files in Keep. This example will use the Python SDK to find public WGS (Whole Genome Sequencing) data for people who have reported a certain medical condition.
*This tutorial assumes that you are "logged into an Arvados VM instance":{{site.baseurl}}/user/getting_started/ssh-access.html#login, and have a "working environment.":{{site.baseurl}}/user/getting_started/check-environment.html*
We query the "links" resource to find humans that report the selected trait. Links are directional connections between Arvados data items, for example, from a human to their reported traits.
<notextile>
-<pre><code>>>> <span class="userinput">trait_query = {
- 'link_class': 'human_trait',
- 'tail_kind': 'arvados#human',
- 'head_uuid': non_melanoma_cancer
- }
+<pre><code>>>> <span class="userinput">trait_filter = [
+ ['link_class', '=', 'human_trait'],
+ ['tail_uuid', 'is_a', 'arvados#human'],
+ ['head_uuid', '=', non_melanoma_cancer],
+ ]
</code></pre>
</notextile>
-* @'link_class'@ queries for links that describe the traits of a particular human.
-* @'tail_kind'@ queries for links where the tail of the link is a human.
-* @'head_uuit'@ queries for links where the head of the link is a specific data item.
+* @['link_class', '=', 'human_trait']@ filters on links that connect phenotype traits to individuals in the database.
+* @['tail_uuid', 'is_a', 'arvados#human']@ filters that the "tail" must be a "human" database object.
+* @['head_uuid', '=', non_melanoma_cancer]@ filters that the "head" of the link must connect to the "trait" database object non_melanoma_cancer .
The query will return links that match all three conditions.
<notextile>
-<pre><code>>>> <span class="userinput">trait_links = arvados.api().links().list(limit=1000, where=trait_query).execute()</span>
+<pre><code>>>> <span class="userinput">trait_links = arvados.api().links().list(limit=1000, filters=trait_filter).execute()</span>
</code></pre>
</notextile>
--- /dev/null
+---
+layout: default
+navsection: userguide
+title: Introduction to Crunch
+...
+
+In "getting data from Keep,":tutorial-keep.html#arv-get we downloaded a file from Keep and did some computation with it (specifically, computing the md5 hash of the complete file). While a straightforward way to accomplish a computational task, there are several obvious drawbacks to this approach:
+* Large files require significant time to download.
+* Very large files may exceed the scratch space of the local disk.
+* We are only able to use the local CPU to process the file.
+
+The Arvados "Crunch" framework is designed to support processing very large data batches (gigabytes to terabytes) efficiently, and provides the following benefits:
+* Increase concurrency by running tasks asynchronously, using many CPUs and network interfaces at once (especially beneficial for CPU-bound and I/O-bound tasks respectively).
+* Track inputs, outputs, and settings so you can verify that the inputs, settings, and sequence of programs you used to arrive at an output is really what you think it was.
+* Ensure that your programs and workflows are repeatable with different versions of your code, OS updates, etc.
+* Interrupt and resume long-running jobs consisting of many short tasks.
+* Maintain timing statistics automatically, so they're there when you want them.
---
layout: default
navsection: userguide
-navmenu: Tutorials
-title: "Running external programs"
-
+title: "Using Crunch to run external programs"
...
-h1. Running external programs
-
This tutorial demonstrates how to use Crunch to run an external program by writting a wrapper using the Python SDK.
*This tutorial assumes that you are "logged into an Arvados VM instance":{{site.baseurl}}/user/getting_started/ssh-access.html#login, and have a "working environment.":{{site.baseurl}}/user/getting_started/check-environment.html*
</code></pre>
</notextile>
-Next, using @nano@ or your favorite Unix text editor, create a new file called @run-md5sum.py@ in the @crunch_scripts@ directory.
+Next, using @nano@ or your favorite Unix text editor, create a new file called @run-md5sum.py@ in the @crunch_scripts@ directory.
notextile. <pre>~/<b>you</b>/crunch_scripts$ <code class="userinput">nano run-md5sum.py</code></pre>
Add the following code to use the @md5sum@ program to compute the hash of each file in a collection:
-<pre><code class="userinput">{% include 'run_md5sum_py' %}</code></pre>
+<notextile> {% code 'run_md5sum_py' as python %} </notextile>
Make the file executable:
notextile. <pre><code>~/<b>you</b>/crunch_scripts$ <span class="userinput">chmod +x run-md5sum.py</span></code></pre>
-Next, add the file to @git@ staging, commit and push:
+Next, use @git@ to stage the file, commit, and push:
<notextile>
<pre><code>~/<b>you</b>/crunch_scripts$ <span class="userinput">git add run-md5sum.py</span>
</code></pre>
</notextile>
-You should now be able to run your new script using Crunch, with "script" referring to our new "run-md5sum.py" script.
+You should now be able to run your new script using Crunch, with @"script"@ referring to our new @run-md5sum.py@ script.
<notextile>
-<pre><code>~/<b>you</b>/crunch_scripts$ <span class="userinput">cat >~/the_job <<EOF
-{
- "script": "run-md5sum.py",
- "script_version": "you:master",
- "script_parameters":
- {
- "input": "c1bad4b39ca5a924e481008009d94e32+210"
- }
-}
-EOF</span>
-~/<b>you</b>/crunch_scripts$ <span class="userinput">arv job create --job "$(cat the_job)"</span>
+<pre><code>~/<b>you</b>/crunch_scripts$ <span class="userinput">cat >~/the_pipeline <<EOF
{
- ...
- "uuid":"qr1hi-xxxxx-xxxxxxxxxxxxxxx"
- ...
-}
-~/<b>you</b>/crunch_scripts$ <span class="userinput">arv job get --uuid qr1hi-xxxxx-xxxxxxxxxxxxxxx</span>
-{
- ...
- "output":"4d164b1658c261b9afc6b479130016a3+54",
- ...
+ "name":"Run external md5sum program",
+ "components":{
+ "do_hash":{
+ "script":"run-md5sum.py",
+ "script_parameters":{
+ "input":{
+ "required": true,
+ "dataclass": "Collection"
+ }
+ },
+ "repository":"$USER",
+ "script_version":"master"
+ }
+ }
}
+EOF
+</span>~/<b>you</b>/crunch_scripts$ <span class="userinput">arv pipeline_template create --pipeline-template "$(cat ~/the_pipeline)"</span>
</code></pre>
</notextile>
+
+(Your shell should automatically fill in @$USER@ with your login name. The JSON that gets saved should have @"repository"@ pointed at your personal git repository.)
+
+Your new pipeline template will appear on the Workbench "Compute %(rarr)→% Pipeline templates":https://{{ site.arvados_workbench_host }}/pipeline_instances page. You can run the "pipeline using Workbench":tutorial-pipeline-workbench.html.
layout: default
navsection: userguide
navmenu: Tutorials
-title: "Writing a Crunch script"
-
+title: "Writing a pipeline"
...
-h1. Writing a Crunch script
-
In this tutorial, we will write the "hash" script demonstrated in the first tutorial.
*This tutorial assumes that you are "logged into an Arvados VM instance":{{site.baseurl}}/user/getting_started/ssh-access.html#login, and have a "working environment.":{{site.baseurl}}/user/getting_started/check-environment.html*
h2. Setting up Git
-As discussed in the previous tutorial, all Crunch scripts are managed through the @git@ revision control system.
-
-First, you should do some basic configuration for git (you only need to do this the first time):
+All Crunch scripts are managed through the @git@ revision control system. Before you start using git, you should do some basic configuration (you only need to do this the first time):
<notextile>
<pre><code>~$ <span class="userinput">git config --global user.name "Your Name"</span>
~$ <span class="userinput">git config --global user.email <b>you</b>@example.com</span></code></pre>
</notextile>
-On the Arvados Workbench, navigate to _Compute %(rarr)→% Code repositories._ You should see two repositories, one named "arvados" (under the *name* column) and a second with your user name. Next to *name* is the column *push_url*. Copy the *push_url* cell associated with your repository. This should look like <notextile><code>git@git.{{ site.arvados_api_host }}:<b>you</b>.git</code></notextile>.
+On the Arvados Workbench, navigate to "Compute %(rarr)→% Code repositories":https://{{site.arvados_workbench_host}}/repositories. You should see a repository with your user name listed in the *name* column. Next to *name* is the column *push_url*. Copy the *push_url* value associated with your repository. This should look like <notextile><code>git@git.{{ site.arvados_api_host }}:<b>you</b>.git</code></notextile>.
Next, on the Arvados virtual machine, clone your git repository:
notextile. <pre><code>$ <span class="userinput">man gittutorial</span></code></pre>
-or <b>"click here to search Google for git tutorials":http://google.com/#q=git+tutorial</b>
+or *"search Google for git tutorials":http://google.com/#q=git+tutorial*.
{% include 'notebox_end' %}
h2. Creating a Crunch script
Add the following code to compute the md5 hash of each file in a collection:
-<pre><code class="userinput">{% include 'tutorial_hash_script_py' %}</code></pre>
+<notextile> {% code 'tutorial_hash_script_py' as python %} </notextile>
Make the file executable:
notextile. <pre><code>~/<b>you</b>/crunch_scripts$ <span class="userinput">chmod +x hash.py</span></code></pre>
{% include 'notebox_begin' %}
-The below steps describe how to execute the script after committing changes to git. To test the script locally, please see the "debugging a crunch script":tutorial-job-debug.html page.
+The steps below describe how to execute the script after committing changes to git. To run a script locally for testing, please see "debugging a crunch script":{{site.baseurl}}/user/topics/tutorial-job-debug.html.
{% include 'notebox_end' %}
-Next, add the file to @git@ staging. This tells @git@ that the file should be included on the next commit.
+Next, add the file to git staging. This tells @git@ that the file should be included on the next commit.
notextile. <pre><code>~/<b>you</b>/crunch_scripts$ <span class="userinput">git add hash.py</span></code></pre>
-Next, commit your changes to git. All staged changes are recorded into the local @git@ repository:
+Next, commit your changes to git. All staged changes are recorded into the local git repository:
<notextile>
<pre><code>~/<b>you</b>/crunch_scripts$ <span class="userinput">git commit -m"my first script"</span>
* [new branch] master -> master</code></pre>
</notextile>
-You should now be able to run your script using Crunch, similar to how we did it in the "first tutorial.":tutorial-job1.html The field @"script_version"@ should be @you:master@ to tell Crunch to run the script at the head of the "master" git branch, which you just uploaded.
+h2. Create a pipeline template
+
+Next, create a file that contains the pipeline definition:
<notextile>
-<pre><code>~/<b>you</b>/crunch_scripts$ <span class="userinput">cat >~/the_job <<EOF
+<pre><code>~/<b>you</b>/crunch_scripts$ <span class="userinput">cd ~</span>
+~$ <span class="userinput">cat >the_pipeline <<EOF
{
- "script": "hash.py",
- "script_version": "you:master",
- "script_parameters":
- {
- "input": "c1bad4b39ca5a924e481008009d94e32+210"
- }
+ "name":"My first pipeline",
+ "components":{
+ "do_hash":{
+ "script":"hash.py",
+ "script_parameters":{
+ "input":{
+ "required": true,
+ "dataclass": "Collection"
+ }
+ },
+ "repository":"$USER",
+ "script_version":"master",
+ "output_is_persistent":true
+ }
+ }
}
-EOF</span>
-~/<b>you</b>/crunch_scripts$ <span class="userinput">arv job create --job "$(cat ~/the_job)"</span>
-{
- ...
- "uuid":"qr1hi-xxxxx-xxxxxxxxxxxxxxx"
- ...
-}
-~/<b>you</b>/crunch_scripts$ <span class="userinput">arv job get --uuid qr1hi-xxxxx-xxxxxxxxxxxxxxx</span>
-{
- ...
- "output":"880b55fb4470b148a447ff38cacdd952+54",
- ...
-}
-~/<b>you</b>/crunch_scripts$ <span class="userinput">arv keep get 880b55fb4470b148a447ff38cacdd952+54/md5sum.txt</span>
-44b8ae3fde7a8a88d2f7ebd237625b4f var-GS000016015-ASM.tsv.bz2
+EOF
+</span></code></pre>
+</notextile>
+
+* @cat@ is a standard Unix utility that writes a sequence of input to standard output.
+* @<<EOF@ tells the shell to direct the following lines into the standard input for @cat@ up until it sees the line @EOF@.
+* @>the_pipeline@ redirects standard output to a file called @the_pipeline@.
+* @"name"@ is a human-readable name for the pipeline.
+* @"components"@ is a set of scripts that make up the pipeline.
+* The component is listed with a human-readable name (@"do_hash"@ in this example).
+* @"repository"@ is the name of a git repository to search for the script version. You can access a list of available git repositories on the Arvados Workbench under "Compute %(rarr)→% Code repositories":https://{{site.arvados_workbench_host}}/repositories. Your shell should automatically fill in @$USER@ with your login name, so that the final JSON has @"repository"@ pointed at your personal git repository.
+* @"script_version"@ specifies the version of the script that you wish to run. This can be in the form of an explicit git revision hash, a tag, or a branch (in which case it will use the HEAD of the specified branch). Arvados logs the script version that was used in the run, enabling you to go back and re-run any past job with the guarantee that the exact same code will be used as was used in the previous run.
+* @"script"@ specifies the filename of the script to run. Crunch expects to find this in the @crunch_scripts/@ subdirectory of the git repository.
+* @"script_parameters"@ describes the parameters for the script. In this example, there is one parameter called @input@ which is @required@ and is a @Collection@.
+* @"output_is_persistent"@ indicates whether the output of the job is considered valuable. If this value is false (or not given), the output will be treated as intermediate data and eventually deleted to reclaim disk space.
+
+Now, use @arv pipeline_template create@ to register your pipeline template in Arvados:
+
+<notextile>
+<pre><code>~$ <span class="userinput">arv pipeline_template create --pipeline-template "$(cat the_pipeline)"</span>
</code></pre>
</notextile>
-Next, "debugging a crunch script.":tutorial-job-debug.html
+Your new pipeline template will appear on the Workbench "Compute %(rarr)→% Pipeline templates":https://{{ site.arvados_workbench_host }}/pipeline_instances page. You can run the "pipeline using Workbench":tutorial-pipeline-workbench.html.
+++ /dev/null
----
-layout: default
-navsection: userguide
-navmenu: Tutorials
-title: "Running a Crunch job"
-
-...
-
-h1. Running a crunch job
-
-This tutorial introduces the concepts and use of the Crunch job system using the @arv@ command line tool and Arvados Workbench.
-
-*This tutorial assumes that you are "logged into an Arvados VM instance":{{site.baseurl}}/user/getting_started/ssh-access.html#login, and have a "working environment.":{{site.baseurl}}/user/getting_started/check-environment.html*
-
-In "retrieving data using Keep,":tutorial-keep.html we downloaded a file from Keep and did some computation with it (specifically, computing the md5 hash of the complete file). While a straightforward way to accomplish a computational task, there are several obvious drawbacks to this approach:
-* Large files require significant time to download.
-* Very large files may exceed the scratch space of the local disk.
-* We are only able to use the local CPU to process the file.
-
-The Arvados "Crunch" framework is designed to support processing very large data batches (gigabytes to terabytes) efficiently, and provides the following benefits:
-* Increase concurrency by running tasks asynchronously, using many CPUs and network interfaces at once (especially beneficial for CPU-bound and I/O-bound tasks respectively).
-* Track inputs, outputs, and settings so you can verify that the inputs, settings, and sequence of programs you used to arrive at an output is really what you think it was.
-* Ensure that your programs and workflows are repeatable with different versions of your code, OS updates, etc.
-* Interrupt and resume long-running jobs consisting of many short tasks.
-* Maintain timing statistics automatically, so they're there when you want them.
-
-For your first job, you will run the "hash" crunch script using the Arvados system. The "hash" script computes the md5 hash of each file in a collection.
-
-Crunch jobs are described using JSON objects. For example:
-
-<notextile>
-<pre><code>~$ <span class="userinput">cat >the_job <<EOF
-{
- "script": "hash",
- "script_version": "arvados:master",
- "script_parameters":
- {
- "input": "c1bad4b39ca5a924e481008009d94e32+210"
- }
-}
-EOF
-</code></pre>
-</notextile>
-
-* @cat@ is a standard Unix utility that simply copies standard input to standard output
-* @<<EOF@ tells the shell to direct the following lines into the standard input for @cat@ up until it sees the line @EOF@
-* @>the_job@ redirects standard output to a file called @the_job@
-* @"script"@ specifies the name of the script to run. The script is searched for in the "crunch_scripts/" subdirectory of the @git@ checkout specified by @"script_version"@.
-* @"script_version"@ specifies the version of the script that you wish to run. This can be in the form of an explicit @git@ revision hash, or in the form "repository:branch" (in which case it will take the HEAD of the specified branch). Arvados logs the script version that was used in the run, enabling you to go back and re-run any past job with the guarantee that the exact same code will be used as was used in the previous run. You can access a list of available @git@ repositories on the Arvados workbench under _Compute %(rarr)→% Code repositories_.
-* @"script_parameters"@ are provided to the script. In this case, the input is the locator for the collection that we inspected in the previous section.
-
-Use @arv job create@ to actually submit the job. It should print out a JSON object which describes the newly created job:
-
-<notextile>
-<pre><code>~$ <span class="userinput">arv job create --job "$(cat the_job)"</span>
-{
- "href":"https://qr1hi.arvadosapi.com/arvados/v1/jobs/qr1hi-8i9sb-1pm1t02dezhupss",
- "kind":"arvados#job",
- "etag":"ax3cn7w9whq2hdh983yxvq09p",
- "uuid":"qr1hi-8i9sb-1pm1t02dezhupss",
- "owner_uuid":"qr1hi-tpzed-9zdpkpni2yddge6",
- "created_at":"2013-12-16T20:44:32Z",
- "modified_by_client_uuid":"qr1hi-ozdt8-obw7foaks3qjyej",
- "modified_by_user_uuid":"qr1hi-tpzed-9zdpkpni2yddge6",
- "modified_at":"2013-12-16T20:44:32Z",
- "updated_at":"2013-12-16T20:44:33Z",
- "submit_id":null,
- "priority":null,
- "script":"hash",
- "script_parameters":{
- "input":"c1bad4b39ca5a924e481008009d94e32+210"
- },
- "script_version":"d9cd657b733d578ac0d2167dd75967aa4f22e0ac",
- "cancelled_at":null,
- "cancelled_by_client_uuid":null,
- "cancelled_by_user_uuid":null,
- "started_at":null,
- "finished_at":null,
- "output":null,
- "success":null,
- "running":null,
- "is_locked_by_uuid":null,
- "log":null,
- "runtime_constraints":{},
- "tasks_summary":{},
- "dependencies":[
- "c1bad4b39ca5a924e481008009d94e32+210"
- ],
- "log_stream_href":"https://qr1hi.arvadosapi.com/arvados/v1/jobs/qr1hi-8i9sb-1pm1t02dezhupss/log_tail_follow"
-}
-</code></pre>
-</notextile>
-
-The job is now queued and will start running as soon as it reaches the front of the queue. Fields to pay attention to include:
-
- * @"uuid"@ is the unique identifier for this specific job
- * @"script_version"@ is the actual revision of the script used. This is useful if the version was described using the "repository:branch" format.
-
-h2. Monitor job progress
-
-Go to the Workbench dashboard. Your job should be at the top of the "Recent jobs" table. This table refreshes automatically. When the job has completed successfully, it will show <span class="label label-success">finished</span> in the *Status* column.
-
-On the command line, you can access log messages while the job runs using @arv job log_tail_follow@:
-
-notextile. <pre><code>~$ <span class="userinput">arv job log_tail_follow --uuid qr1hi-8i9sb-xxxxxxxxxxxxxxx</span></code></pre>
-
-This will print out the last several lines of the log for that job.
-
-h2. Inspect the job output
-
-On the workbench dashboard, look for the *Output* column of the *Recent jobs* table. Click on the link under *Output* for your job to go to the files page with the job output. The files page lists all the files that were output by the job. Click on the link under the *files* column to view a file, or click on the download icon <span class="glyphicon glyphicon-download-alt"></span> to download the output file.
-
-On the command line, you can use @arv job get@ to access a JSON object describing the output:
-
-<notextile>
-<pre><code>~$ <span class="userinput">arv job get --uuid qr1hi-8i9sb-xxxxxxxxxxxxxxx</span>
-{
- "href":"https://qr1hi.arvadosapi.com/arvados/v1/jobs/qr1hi-8i9sb-1pm1t02dezhupss",
- "kind":"arvados#job",
- "etag":"1bk98tdj0qipjy0rvrj03ta5r",
- "uuid":"qr1hi-8i9sb-1pm1t02dezhupss",
- "owner_uuid":"qr1hi-tpzed-9zdpkpni2yddge6",
- "created_at":"2013-12-16T20:44:32Z",
- "modified_by_client_uuid":null,
- "modified_by_user_uuid":"qr1hi-tpzed-9zdpkpni2yddge6",
- "modified_at":"2013-12-16T20:44:55Z",
- "updated_at":"2013-12-16T20:44:55Z",
- "submit_id":null,
- "priority":null,
- "script":"hash",
- "script_parameters":{
- "input":"c1bad4b39ca5a924e481008009d94e32+210"
- },
- "script_version":"d9cd657b733d578ac0d2167dd75967aa4f22e0ac",
- "cancelled_at":null,
- "cancelled_by_client_uuid":null,
- "cancelled_by_user_uuid":null,
- "started_at":"2013-12-16T20:44:36Z",
- "finished_at":"2013-12-16T20:44:53Z",
- "output":"880b55fb4470b148a447ff38cacdd952+54",
- "success":true,
- "running":false,
- "is_locked_by_uuid":"qr1hi-tpzed-9zdpkpni2yddge6",
- "log":"2afdc6c8b67372ffd22d8ce89d35411f+91",
- "runtime_constraints":{},
- "tasks_summary":{
- "done":2,
- "running":0,
- "failed":0,
- "todo":0
- },
- "dependencies":[
- "c1bad4b39ca5a924e481008009d94e32+210"
- ],
- "log_stream_href":null
-}
-</code></pre>
-</notextile>
-
-* @"output"@ is the unique identifier for this specific job's output. This is a Keep collection. Because the output of Arvados jobs should be deterministic, the known expected output is <code>880b55fb4470b148a447ff38cacdd952+54</code>.
-
-Now you can list the files in the collection:
-
-<notextile>
-<pre><code>~$ <span class="userinput">arv keep ls 880b55fb4470b148a447ff38cacdd952+54</span>
-md5sum.txt
-</code></pre>
-</notextile>
-
-This collection consists of the @md5sum.txt@ file. Use @arv keep get@ to show the contents of the @md5sum.txt@ file:
-
-<notextile>
-<pre><code>~$ <span class="userinput">arv keep get 880b55fb4470b148a447ff38cacdd952+54/md5sum.txt</span>
-44b8ae3fde7a8a88d2f7ebd237625b4f var-GS000016015-ASM.tsv.bz2
-</code></pre>
-</notextile>
-
-This md5 hash matches the md5 hash which we computed earlier.
-
-h2. The job log
-
-When the job completes, you can access the job log. On the workbench dashboard, this is the link under the *Log* column of the *Recent jobs* table.
-
-On the command line, the keep identifier listed in the @"log"@ field from @arv job get@ specifies a collection. You can list the files in the collection:
-
-<notextile>
-<pre><code>~$ <span class="userinput">arv keep ls xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx+91</span>
-qr1hi-8i9sb-xxxxxxxxxxxxxxx.log.txt
-</code></pre>
-</notextile>
-
-The log collection consists of one log file named with the job id. You can access it using @arv keep get@:
-
-<notextile>
-<pre><code>~$ <span class="userinput">arv keep get xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx+91/qr1hi-8i9sb-xxxxxxxxxxxxxxx.log.txt</span>
-2013-12-16_20:44:35 qr1hi-8i9sb-1pm1t02dezhupss 7575 check slurm allocation
-2013-12-16_20:44:35 qr1hi-8i9sb-1pm1t02dezhupss 7575 node compute13 - 8 slots
-2013-12-16_20:44:36 qr1hi-8i9sb-1pm1t02dezhupss 7575 start
-2013-12-16_20:44:36 qr1hi-8i9sb-1pm1t02dezhupss 7575 Install revision d9cd657b733d578ac0d2167dd75967aa4f22e0ac
-2013-12-16_20:44:37 qr1hi-8i9sb-1pm1t02dezhupss 7575 Clean-work-dir exited 0
-2013-12-16_20:44:37 qr1hi-8i9sb-1pm1t02dezhupss 7575 Install exited 0
-2013-12-16_20:44:37 qr1hi-8i9sb-1pm1t02dezhupss 7575 script hash
-2013-12-16_20:44:37 qr1hi-8i9sb-1pm1t02dezhupss 7575 script_version d9cd657b733d578ac0d2167dd75967aa4f22e0ac
-2013-12-16_20:44:37 qr1hi-8i9sb-1pm1t02dezhupss 7575 script_parameters {"input":"c1bad4b39ca5a924e481008009d94e32+210"}
-2013-12-16_20:44:37 qr1hi-8i9sb-1pm1t02dezhupss 7575 runtime_constraints {"max_tasks_per_node":0}
-2013-12-16_20:44:37 qr1hi-8i9sb-1pm1t02dezhupss 7575 start level 0
-2013-12-16_20:44:37 qr1hi-8i9sb-1pm1t02dezhupss 7575 status: 0 done, 0 running, 1 todo
-2013-12-16_20:44:38 qr1hi-8i9sb-1pm1t02dezhupss 7575 0 job_task qr1hi-ot0gb-23c1k3kwrf8da62
-2013-12-16_20:44:38 qr1hi-8i9sb-1pm1t02dezhupss 7575 0 child 7681 started on compute13.1
-
-2013-12-16_20:44:38 qr1hi-8i9sb-1pm1t02dezhupss 7575 status: 0 done, 1 running, 0 todo
-2013-12-16_20:44:39 qr1hi-8i9sb-1pm1t02dezhupss 7575 0 child 7681 on compute13.1 exit 0 signal 0 success=true
-2013-12-16_20:44:39 qr1hi-8i9sb-1pm1t02dezhupss 7575 0 success in 1 seconds
-2013-12-16_20:44:39 qr1hi-8i9sb-1pm1t02dezhupss 7575 0 output
-2013-12-16_20:44:39 qr1hi-8i9sb-1pm1t02dezhupss 7575 wait for last 0 children to finish
-2013-12-16_20:44:39 qr1hi-8i9sb-1pm1t02dezhupss 7575 status: 1 done, 0 running, 1 todo
-2013-12-16_20:44:39 qr1hi-8i9sb-1pm1t02dezhupss 7575 start level 1
-2013-12-16_20:44:39 qr1hi-8i9sb-1pm1t02dezhupss 7575 status: 1 done, 0 running, 1 todo
-2013-12-16_20:44:39 qr1hi-8i9sb-1pm1t02dezhupss 7575 1 job_task qr1hi-ot0gb-iwr0o3unqothg28
-2013-12-16_20:44:39 qr1hi-8i9sb-1pm1t02dezhupss 7575 1 child 7716 started on compute13.1
-2013-12-16_20:44:39 qr1hi-8i9sb-1pm1t02dezhupss 7575 status: 1 done, 1 running, 0 todo
-2013-12-16_20:44:52 qr1hi-8i9sb-1pm1t02dezhupss 7575 1 child 7716 on compute13.1 exit 0 signal 0 success=true
-2013-12-16_20:44:52 qr1hi-8i9sb-1pm1t02dezhupss 7575 1 success in 13 seconds
-2013-12-16_20:44:52 qr1hi-8i9sb-1pm1t02dezhupss 7575 1 output 880b55fb4470b148a447ff38cacdd952+54
-2013-12-16_20:44:52 qr1hi-8i9sb-1pm1t02dezhupss 7575 wait for last 0 children to finish
-2013-12-16_20:44:52 qr1hi-8i9sb-1pm1t02dezhupss 7575 status: 2 done, 0 running, 0 todo
-2013-12-16_20:44:52 qr1hi-8i9sb-1pm1t02dezhupss 7575 release job allocation
-2013-12-16_20:44:52 qr1hi-8i9sb-1pm1t02dezhupss 7575 Freeze not implemented
-2013-12-16_20:44:52 qr1hi-8i9sb-1pm1t02dezhupss 7575 collate
-2013-12-16_20:44:53 qr1hi-8i9sb-1pm1t02dezhupss 7575 output 880b55fb4470b148a447ff38cacdd952+54
-2013-12-16_20:44:53 qr1hi-8i9sb-1pm1t02dezhupss 7575 finish
-</code></pre>
-</notextile>
-
-This concludes the first tutorial. In the next tutorial, we will "write a script to compute the hash.":tutorial-firstscript.html
---
layout: default
navsection: userguide
-navmenu: Tutorials
-title: "Storing and Retrieving data using Arvados Keep"
-
+title: "Storing and Retrieving data using Keep"
...
-h1. Storing and Retrieving data using Arvados Keep
-
This tutorial introduces you to the Arvados file storage system.
The Arvados distributed file system is called *Keep*. Keep is a content-addressable file system. This means that files are managed using special unique identifiers derived from the _contents_ of the file, rather than human-assigned file names (specifically, the md5 hash). This has a number of advantages:
* Files can be stored and replicated across a cluster of servers without requiring a central name server.
-* Systematic validation of data integrity by both server and client because the checksum is built into the identifier.
-* Minimizes data duplication (two files with the same contents will result in the same identifier, and will not be stored twice.)
-* Avoids data race conditions (an identifier always points to the same data.)
+* Both the server and client systematically validate data integrity because the checksum is built into the identifier.
+* Data duplication is minimized—two files with the same contents will have in the same identifier, and will not be stored twice.
+* It avoids data race conditions, since an identifier always points to the same data.
h1. Putting Data into Keep
-We will start with downloading a freely available VCF file from the "Personal Genome Project (PGP)":http://www.personalgenomes.org subject "hu599905":https://my.personalgenomes.org/profile/hu599905 to a staging directory on the VM, and then add it to Keep.
+We will start by downloading a freely available VCF file from "Personal Genome Project (PGP)":http://www.personalgenomes.org subject "hu599905":https://my.personalgenomes.org/profile/hu599905 to a staging directory on the VM, and adding it to Keep. In the following commands, replace *@you@* with your login name.
-In the following tutorials, replace <b><code>you</code></b> with your user id.
-
-First, log into the Arvados VM instance and set up the staging area:
+First, log into your Arvados VM and set up the staging area:
notextile. <pre><code>~$ <span class="userinput">mkdir /scratch/<b>you</b></span></code></pre>
The output value @c1bad4b39ca5a924e481008009d94e32+210@ from @arv keep put@ is the Keep locator. This enables you to access the file you just uploaded, and is explained in the next section.
-h2. Putting a directory
+h2(#dir). Putting a directory
You can also use @arv keep put@ to add an entire directory:
/scratch/<b>you</b>$ <span class="userinput">echo "hello bob" > tmp/bob.txt</span>
/scratch/<b>you</b>$ <span class="userinput">echo "hello carol" > tmp/carol.txt</span>
/scratch/<b>you</b>$ <span class="userinput">arv keep put tmp</span>
-0M / 0M 100.0%
+0M / 0M 100.0%
887cd41e9c613463eab2f0d885c6dd96+83
</code></pre>
</notextile>
+The locator @887cd41e9c613463eab2f0d885c6dd96+83@ represents a collection with multiple files.
+
h1. Getting Data from Keep
-In Keep, information is stored in *data blocks*. Data blocks are normally between 1 byte and 64 megabytes in size. If a file exceeds the maximum size of a single data block, the file will be split across multiple data blocks until the entire file can be stored. These data blocks may be stored and replicated across multiple disks, servers, or clusters. Each data block has its own identifier for the contents of that specific data block.
+h2. Using Workbench
-In order to reassemble the file, Keep stores a *collection* data block which lists in sequence the data blocks that make up the original file. A collection data block may store the information for multiple files, including a directory structure.
+You may access collections through the "Collections section of Arvados Workbench":https://{{ site.arvados_workbench_host }}/collections at *Data* %(rarr)→% *Collections (data files)*. You can also access individual files within a collection. Some examples:
-In this example we will use @c1bad4b39ca5a924e481008009d94e32+210@ which we added to keep in the previous section. First let us examine the contents of this collection using @arv keep get@:
+* "https://{{ site.arvados_workbench_host }}/collections/c1bad4b39ca5a924e481008009d94e32+210":https://{{ site.arvados_workbench_host }}/collections/c1bad4b39ca5a924e481008009d94e32+210
+* "https://{{ site.arvados_workbench_host }}/collections/887cd41e9c613463eab2f0d885c6dd96+83/alice.txt":https://{{ site.arvados_workbench_host }}/collections/887cd41e9c613463eab2f0d885c6dd96+83/alice.txt
+
+h2(#arv-get). Using the command line
+
+You can view the contents of a collection using @arv keep ls@:
<notextile>
-<pre><code>/scratch/<b>you</b>$ <span class="userinput">arv keep get c1bad4b39ca5a924e481008009d94e32+210</span>
-. 204e43b8a1185621ca55a94839582e6f+67108864 b9677abbac956bd3e86b1deb28dfac03+67108864 fc15aff2a762b13f521baf042140acec+67108864 323d2a3ce20370c4ca1d3462a344f8fd+25885655 0:227212247:var-GS000016015-ASM.tsv.bz2
+<pre><code>/scratch/<b>you</b>$ <span class="userinput">arv keep ls c1bad4b39ca5a924e481008009d94e32+210</span>
+var-GS000016015-ASM.tsv.bz2
</code></pre>
-</notextile>
-The command @arv keep get@ fetches the contents of the locator @c1bad4b39ca5a924e481008009d94e32+210@. This is a locator for a collection data block, so it fetches the contents of the collection. In this example, this collection consists of a single file @var-GS000016015-ASM.tsv.bz2@ which is 227212247 bytes long, and is stored using four sequential data blocks, <code>204e43b8a1185621ca55a94839582e6f+67108864</code>, <code>b9677abbac956bd3e86b1deb28dfac03+67108864</code>, <code>fc15aff2a762b13f521baf042140acec+67108864</code>, <code>323d2a3ce20370c4ca1d3462a344f8fd+25885655</code>.
+<pre><code>/scratch/<b>you</b>$ <span class="userinput">arv keep ls 887cd41e9c613463eab2f0d885c6dd96+83</span>
+alice.txt
+bob.txt
+carol.txt
+</code></pre>
+</notextile>
-Let's use @arv keep get@ to download the first datablock:
+Use @-s@ to print file sizes rounded up to the nearest kilobyte:
-notextile. <pre><code>/scratch/<b>you</b>$ <span class="userinput">arv keep get 204e43b8a1185621ca55a94839582e6f+67108864 > block1</span></code></pre>
+<notextile>
+<pre><code>/scratch/<b>you</b>$ <span class="userinput">arv keep ls -s c1bad4b39ca5a924e481008009d94e32+210</span>
+221887 var-GS000016015-ASM.tsv.bz2
+</code></pre>
+</notextile>
-Let's look at the size and compute the md5 hash of @block1@:
+Use @arv keep get@ to download the contents of a collection and place it in the directory specified in the second argument (in this example, @.@ for the current directory):
<notextile>
-<pre><code>/scratch/<b>you</b>$ <span class="userinput">ls -l block1</span>
--rw-r--r-- 1 you group 67108864 Dec 9 20:14 block1
-/scratch/<b>you</b>$ <span class="userinput">md5sum block1</span>
-204e43b8a1185621ca55a94839582e6f block1
+<pre><code>/scratch/<b>you</b>$ <span class="userinput">arv keep get c1bad4b39ca5a924e481008009d94e32+210/ .</span>
+/scratch/<b>you</b>$ <span class="userinput">ls var-GS000016015-ASM.tsv.bz2</span>
+var-GS000016015-ASM.tsv.bz2
</code></pre>
</notextile>
-Notice that the block identifer <code>204e43b8a1185621ca55a94839582e6f+67108864</code> of:
-* the md5 hash @204e43b8a1185621ca55a94839582e6f@ which matches the md5 hash of @block1@
-* a size hint @67108864@ which matches the size of @block1@
-
-Next, let's use @arv keep get@ to download and reassemble @var-GS000016015-ASM.tsv.bz2@ using the following command:
+You can also download individual files:
<notextile>
-<pre><code>/scratch/<b>you</b>$ <span class="userinput">arv keep get c1bad4b39ca5a924e481008009d94e32+210/var-GS000016015-ASM.tsv.bz2 .</span>
+<pre><code>/scratch/<b>you</b>$ <span class="userinput">arv keep get 887cd41e9c613463eab2f0d885c6dd96+83/alice.txt .</span>
</code></pre>
+</notextile>
-This downloads the file @var-GS000016015-ASM.tsv.bz2@ described by collection @c1bad4b39ca5a924e481008009d94e32+210@ from Keep and places it into the local directory. Now that we have the file, we can compute the md5 hash of the complete file:
+With a local copy of the file, we can do some computation, for example computing the md5 hash of the complete file:
<notextile>
<pre><code>/scratch/<b>you</b>$ <span class="userinput">md5sum var-GS000016015-ASM.tsv.bz2</span>
</code></pre>
</notextile>
-h2. Accessing Collections
+h2. Using arv-mount
-There are a couple of other ways to access a collection. You may view the contents of a collection using @arv keep ls@:
+Use @arv-mount@ to mount a Keep collection and access it using traditional filesystem tools.
<notextile>
-<pre><code>/scratch/<b>you</b>$ <span class="userinput">arv keep ls c1bad4b39ca5a924e481008009d94e32+210</span>
+<pre><code>/scratch/<b>you</b>$ <span class="userinput">mkdir -p mnt</span>
+/scratch/<b>you</b>$ <span class="userinput">arv-mount --collection c1bad4b39ca5a924e481008009d94e32+210 mnt &</span>
+/scratch/<b>you</b>$ <span class="userinput">cd mnt</span>
+/scratch/<b>you</b>/mnt$ <span class="userinput">ls</span>
var-GS000016015-ASM.tsv.bz2
-/scratch/<b>you</b>$ <span class="userinput">arv keep ls -s c1bad4b39ca5a924e481008009d94e32+210</span>
-221887 var-GS000016015-ASM.tsv.bz2
+/scratch/<b>you</b>/mnt$ <span class="userinput">md5sum var-GS000016015-ASM.tsv.bz2</span>
+44b8ae3fde7a8a88d2f7ebd237625b4f var-GS000016015-ASM.tsv.bz2
+/scratch/<b>you</b>/mnt$ <span class="userinput">cd ..</span>
+/scratch/<b>you</b>$ <span class="userinput">fusermount -u mnt</span>
</code></pre>
</notextile>
-* @-s@ prints file sizes in kilobytes
+You can also mount the entire Keep namespace in "magic directory" mode:
-You may also access through the Arvados Workbench using a URI similar to this, where the last part of the path is the Keep locator:
+<notextile>
+<pre><code>/scratch/<b>you</b>$ <span class="userinput">mkdir -p mnt</span>
+/scratch/<b>you</b>$ <span class="userinput">arv-mount mnt &</span>
+/scratch/<b>you</b>$ <span class="userinput">cd mnt/c1bad4b39ca5a924e481008009d94e32+210</span>
+/scratch/<b>you</b>/mnt/c1bad4b39ca5a924e481008009d94e32+210$ <span class="userinput">ls</span>
+var-GS000016015-ASM.tsv.bz2
+/scratch/<b>you</b>/mnt/c1bad4b39ca5a924e481008009d94e32+210$ <span class="userinput">md5sum var-GS000016015-ASM.tsv.bz2</span>
+44b8ae3fde7a8a88d2f7ebd237625b4f var-GS000016015-ASM.tsv.bz2
+/scratch/<b>you</b>/mnt/c1bad4b39ca5a924e481008009d94e32+210$ <span class="userinput">cd ../..</span>
+/scratch/<b>you</b>$ <span class="userinput">fusermount -u mnt</span>
+</code></pre>
+</notextile>
-"https://workbench.{{ site.arvados_api_host }}/collections/c1bad4b39ca5a924e481008009d94e32+210":https://workbench.{{ site.arvados_api_host }}/collections/c1bad4b39ca5a924e481008009d94e32+210
+@arv-mount@ provides several features:
-You are now ready to proceed to the next tutorial, "running a crunch job.":tutorial-job1.html
+* You can browse, open and read Keep entries as if they are regular files.
+* It is easy for existing tools to access files in Keep.
+* Data is downloaded on demand. It is not necessary to download an entire file or collection to start processing.
---
layout: default
navsection: userguide
-navmenu: Tutorials
-title: "Constructing a Crunch pipeline"
-
+title: "Writing a multi-step pipeline"
...
-h1. Constructing a Crunch pipeline
-
A pipeline in Arvados is a collection of crunch scripts, in which the output from one script may be used as the input to another script.
*This tutorial assumes that you are "logged into an Arvados VM instance":{{site.baseurl}}/user/getting_started/ssh-access.html#login, and have a "working environment.":{{site.baseurl}}/user/getting_started/check-environment.html*
+This tutorial uses *@you@* to denote your username. Replace *@you@* with your user name in all the following examples.
+
h2. Create a new script
-Our second script will filter the output of @parallel_hash.py@ and only include hashes that start with 0. Create a new script in @crunch_scripts/@ called @0-filter.py@:
+Our second script will filter the output of @hash.py@ and only include hashes that start with 0. Create a new script in <notextile><code>~/<b>you</b>/crunch_scripts/</code></notextile> called @0-filter.py@:
-<pre><code class="userinput">{% include '0_filter_py' %}</code></pre>
+<notextile> {% code '0_filter_py' as python %} </notextile>
Now add it to git:
<notextile>
-<pre><code>$ <span class="userinput">git add 0-filter.py</span>
-$ <span class="userinput">git commit -m"zero filter"</span>
-$ <span class="userinput">git push origin master</span>
+<pre><code>~/<b>you</b>/crunch_scripts$ <span class="userinput">chmod +x 0-filter.py</span>
+~/<b>you</b>/crunch_scripts$ <span class="userinput">git add 0-filter.py</span>
+~/<b>you</b>/crunch_scripts$ <span class="userinput">git commit -m"zero filter"</span>
+~/<b>you</b>/crunch_scripts$ <span class="userinput">git push origin master</span>
</code></pre>
</notextile>
Next, create a file that contains the pipeline definition:
<notextile>
-<pre><code>$ <span class="userinput">cat >the_pipeline <<EOF
+<pre><code>~/<b>you</b>/crunch_scripts$ <span class="userinput">cat >~/the_pipeline <<EOF
{
- "name":"my_first_pipeline",
+ "name":"Filter md5 hash values",
"components":{
"do_hash":{
- "script":"parallel-hash.py",
+ "script":"hash.py",
"script_parameters":{
- "input": "887cd41e9c613463eab2f0d885c6dd96+83"
+ "input":{
+ "required": true,
+ "dataclass": "Collection"
+ }
},
- "script_version":"you:master"
+ "repository":"$USER",
+ "script_version":"master",
+ "output_is_persistent":false
},
- "filter":{
+ "do_filter":{
"script":"0-filter.py",
"script_parameters":{
"input":{
"output_of":"do_hash"
}
},
- "script_version":"you:master"
+ "repository":"$USER",
+ "script_version":"master",
+ "output_is_persistent":true
}
}
}
EOF
-</code></pre>
+</span></code></pre>
</notextile>
-* @"name"@ is a human-readable name for the pipeline
-* @"components"@ is a set of scripts that make up the pipeline
-* Each component is listed with a human-readable name (@"do_hash"@ and @"filter"@ in this example)
-* Each item in @"components"@ is a single Arvados job, and uses the same format that we saw previously with @arv job create@
-* @"output_of"@ indicates that the @"input"@ of @"filter"@ is the @"output"@ of the @"do_hash"@ component. This is a _dependency_. Arvados uses the dependencies between jobs to automatically determine the correct order to run the jobs.
-
-Now, use @arv pipeline_template create@ tell Arvados about your pipeline template:
-
-<notextile>
-<pre><code>$ <span class="userinput">arv pipeline_template create --pipeline-template "$(cat the_pipeline)"</span>
-qr1hi-p5p6p-xxxxxxxxxxxxxxx
-</code></pre>
-</notextile>
-
-Your new pipeline template will appear on the Workbench %(rarr)→% Compute %(rarr)→% Pipeline templates page.
-
-h3. Running a pipeline
-
-Run the pipeline using @arv pipeline run@, using the UUID that you received from @arv pipeline create@:
-
-<notextile>
-<pre><code>$ <span class="userinput">arv pipeline run --template qr1hi-p5p6p-xxxxxxxxxxxxxxx</span>
-2013-12-16 14:08:40 +0000 -- pipeline_instance qr1hi-d1hrv-vxzkp38nlde9yyr
-do_hash qr1hi-8i9sb-hoyc2u964ecv1s6 queued 2013-12-16T14:08:40Z
-filter - -
-2013-12-16 14:08:51 +0000 -- pipeline_instance qr1hi-d1hrv-vxzkp38nlde9yyr
-do_hash qr1hi-8i9sb-hoyc2u964ecv1s6 e2ccd204bca37c77c0ba59fc470cd0f7+162
-filter qr1hi-8i9sb-w5k40fztqgg9i2x queued 2013-12-16T14:08:50Z
-2013-12-16 14:09:01 +0000 -- pipeline_instance qr1hi-d1hrv-vxzkp38nlde9yyr
-do_hash qr1hi-8i9sb-hoyc2u964ecv1s6 e2ccd204bca37c77c0ba59fc470cd0f7+162
-filter qr1hi-8i9sb-w5k40fztqgg9i2x 735ac35adf430126cf836547731f3af6+56
-</code></pre>
-</notextile>
-
-This instantiates your pipeline and displays a live feed of its status. The new pipeline instance will also show up on the Workbench %(rarr)→% Compute %(rarr)→% Pipeline instances page.
-
-Arvados adds each pipeline component to the job queue as its dependencies are satisfied (or immediately if it has no dependencies) and finishes when all components are completed or failed and there is no more work left to do.
-
-The Keep locators of the output of each of @"do_hash"@ and @"filter"@ component are available from the output log shown above. The output is also available on the Workbench by navigating to %(rarr)→% Compute %(rarr)→% Pipeline instances %(rarr)→% pipeline uuid under the *id* column %(rarr)→% components.
+* @"output_of"@ indicates that the @output@ of the @do_hash@ component should be used as the @"input"@ of @do_filter@. Arvados uses these dependencies between jobs to automatically determine the correct order to run them.
-<notextile>
-<pre><code>$ <span class="userinput">arv keep get e2ccd204bca37c77c0ba59fc470cd0f7+162/md5sum.txt</span>
-0f1d6bcf55c34bed7f92a805d2d89bbf alice.txt
-504938460ef369cd275e4ef58994cffe bob.txt
-8f3b36aff310e06f3c5b9e95678ff77a carol.txt
-$ <span class="userinput">arv keep get 735ac35adf430126cf836547731f3af6+56</span>
-0f1d6bcf55c34bed7f92a805d2d89bbf alice.txt
-</code></pre>
-</notextile>
-
-Indeed, the filter has picked out just the "alice" file as having a hash that starts with 0.
-
-h3. Running a pipeline with different parameters
-
-Notice that the pipeline definition explicitly specifies the Keep locator for the input:
-
-<notextile>
-<pre><code>...
- "do_hash":{
- "script_parameters":{
- "input": "887cd41e9c613463eab2f0d885c6dd96+83"
- },
- }
-...
-</code></pre>
-</notextile>
-
-What if we want to run the pipeline on a different input block? One option is to define a new pipeline template, but would potentially result in clutter with many pipeline templates defined for one-off jobs. Instead, you can override values in the input of the component like this:
-
-<notextile>
-<pre><code>$ <span class="userinput">arv pipeline run --template qr1hi-d1hrv-vxzkp38nlde9yyr do_hash::input=33a9f3842b01ea3fdf27cc582f5ea2af+242</span>
-2013-12-17 20:31:24 +0000 -- pipeline_instance qr1hi-d1hrv-tlkq20687akys8e
-do_hash qr1hi-8i9sb-rffhuay4jryl2n2 queued 2013-12-17T20:31:24Z
-filter - -
-2013-12-17 20:31:34 +0000 -- pipeline_instance qr1hi-d1hrv-tlkq20687akys8e
-do_hash qr1hi-8i9sb-rffhuay4jryl2n2 {:done=>1, :running=>1, :failed=>0, :todo=>0}
-filter - -
-2013-12-17 20:31:44 +0000 -- pipeline_instance qr1hi-d1hrv-tlkq20687akys8e
-do_hash qr1hi-8i9sb-rffhuay4jryl2n2 {:done=>1, :running=>1, :failed=>0, :todo=>0}
-filter - -
-2013-12-17 20:31:55 +0000 -- pipeline_instance qr1hi-d1hrv-tlkq20687akys8e
-do_hash qr1hi-8i9sb-rffhuay4jryl2n2 880b55fb4470b148a447ff38cacdd952+54
-filter qr1hi-8i9sb-j347g1sqovdh0op queued 2013-12-17T20:31:55Z
-2013-12-17 20:32:05 +0000 -- pipeline_instance qr1hi-d1hrv-tlkq20687akys8e
-do_hash qr1hi-8i9sb-rffhuay4jryl2n2 880b55fb4470b148a447ff38cacdd952+54
-filter qr1hi-8i9sb-j347g1sqovdh0op fb728f0ffe152058fa64b9aeed344cb5+54
-</code></pre>
-</notextile>
+(Your shell should automatically fill in @$USER@ with your login name. The JSON that gets saved should have @"repository"@ pointed at your personal git repository.)
-Now check the output:
+Now, use @arv pipeline_template create@ to register your pipeline template in Arvados:
<notextile>
-<pre><code>$ <span class="userinput">arv keep ls -s fb728f0ffe152058fa64b9aeed344cb5+54</span>
-0 0-filter.txt
+<pre><code>~/<b>you</b>/crunch_scripts$ <span class="userinput">arv pipeline_template create --pipeline-template "$(cat ~/the_pipeline)"</span>
</code></pre>
</notextile>
-Here the filter script output is empty, so none of the files in the collection have hash code that start with 0.
+Your new pipeline template will appear on the Workbench "Compute %(rarr)→% Pipeline templates":https://{{ site.arvados_workbench_host }}/pipeline_instances page.
--- /dev/null
+---
+layout: default
+navsection: userguide
+title: "Running a pipeline using Workbench"
+...
+
+notextile. <div class="spaced-out">
+
+# Go to "Collections":https://{{ site.arvados_workbench_host }}/collections (*Data* %(rarr)→% *Collections (data files)*).
+# On the Collections page, go to the search box <span class="glyphicon glyphicon-search"></span> and search for "tutorial".
+# The results should include a collection with the contents *var-GS000016015-ASM.tsv.bz2*.
+# Click on the check box to the left of *var-GS000016015-ASM.tsv.bz2*. This puts the collection in your persistent selection list. You can click on the paperclip <span class="glyphicon glyphicon-paperclip"></span> in the upper right to review your current selections.
+# Go to "Pipeline templates":https://{{ site.arvados_workbench_host }}/pipeline_templates (*Compute* %(rarr)→% *Pipeline templates*).
+# Look for a pipeline named *Tutorial pipeline*.
+# Click on the play button <span class="glyphicon glyphicon-play"></span> to the left of *Tutorial pipeline*. This will take you to a new page to configure the pipeline.
+# Under the *parameter* column, look for *input*. Set the value of *input* by clicking on *none* to get a selection popup. The collection that you selected in step 4 will be at the top of that pulldown menu. Select that collection in the pulldown menu.
+# You can now click on the *Run pipeline* button in the upper right to start the pipeline. A new page shows the pipeline status, queued to run.
+# The page refreshes automatically every 15 seconds. You should see the pipeline running, and then finish successfully.
+# Once the pipeline is finished, click on the link under the *output* column. This will take you to the collection page for the output of this pipeline.
+# Click on *md5sum.txt* to see the actual file that is the output of this pipeline.
+# Go back to the collection page for the result. Click on the *Provenance graph* tab to see a graph illustrating the collections and scripts that were used to generate this file.
+
+notextile. </div>
+
require 'zenweb'
+require 'liquid'
module ZenwebLiquid
VERSION = '0.0.1'
##
# Render a page's liquid and return the intermediate result
def liquid template, content, page, binding = TOPLEVEL_BINDING
- require 'liquid'
Liquid::Template.file_system = Liquid::LocalFileSystem.new(File.join(File.dirname(Rake.application().rakefile), "_includes"))
unless defined? @liquid_template
@liquid_template = Liquid::Template.parse(template)
@liquid_template.render(vars)
end
end
+
+ class LiquidCode < Liquid::Include
+ Syntax = /(#{Liquid::QuotedFragment}+)(\s+(?:as)\s+(#{Liquid::QuotedFragment}+))?/o
+
+ def initialize(tag_name, markup, tokens)
+ Liquid::Tag.instance_method(:initialize).bind(self).call(tag_name, markup, tokens)
+
+ if markup =~ Syntax
+ @template_name = $1
+ @language = $3
+ @attributes = {}
+ else
+ raise SyntaxError.new("Error in tag 'code' - Valid syntax: include '[code_file]' as '[language]'")
+ end
+ end
+
+ def render(context)
+ require 'coderay'
+
+ partial = load_cached_partial(context)
+ html = ''
+
+ context.stack do
+ html = CodeRay.scan(partial.root.nodelist.join, @language).div
+ end
+
+ html
+ end
+
+ Liquid::Template.register_tag('code', LiquidCode)
+ end
end
none /cgroup cgroup defaults 0 0
$ sudo mount /cgroup
</pre>
-
- 3. Enable IPv4 forwarding:
-
- <pre>
- $ grep ipv4.ip_forward /etc/sysctl.conf
- net.ipv4.ip_forward=1
- $ sudo sysctl net.ipv4.ip_forward=1
- </pre>
- 4. [Download and run a docker binary from docker.io.](http://docs.docker.io/en/latest/installation/binaries/)
+ 3. [Download and run a docker binary from docker.io.](http://docs.docker.io/en/latest/installation/binaries/)
-* Ruby (any version)
+* Ruby (version 1.9.3 or greater)
* sudo privileges to run `debootstrap`
Building
--------
-1. Copy `config.yml.example` to `config.yml` and edit it with settings
- for your installation.
-2. Run `make` to build the following Docker images:
+Type `./build.sh` to configure and build the following Docker images:
* arvados/api - the Arvados API server
* arvados/doc - Arvados documentation
* arvados/workbench - the Arvados console
* arvados/sso - the Arvados single-signon authentication server
- You may also build Docker images for individual Arvados services:
-
- $ make api-image
- $ make doc-image
- $ make warehouse-image
- $ make workbench-image
- $ make sso-image
+`build.sh` will generate reasonable defaults for all configuration
+settings. If you want more control over the way Arvados is
+configured, first copy `config.yml.example` to `config.yml` and edit
+it with appropriate configuration settings, and then run `./build.sh`.
Running
-------
RUN apt-get update && \
apt-get -q -y install procps postgresql postgresql-server-dev-9.1 apache2 \
supervisor && \
- git clone git://github.com/curoverse/arvados.git /var/cache/git/arvados.git
+ git clone --bare git://github.com/curoverse/arvados.git /var/cache/git/arvados.git
RUN /bin/mkdir -p /usr/src/arvados/services
ADD generated/api.tar.gz /usr/src/arvados/services/
ENV RAILS_ENV production
ADD generated/config_databases.sh /tmp/config_databases.sh
ADD generated/superuser_token /tmp/superuser_token
-RUN sh /tmp/config_databases.sh && \
+RUN bundle install --gemfile=/usr/src/arvados/services/api/Gemfile && \
+ sh /tmp/config_databases.sh && \
rm /tmp/config_databases.sh && \
/etc/init.d/postgresql start && \
cd /usr/src/arvados/services/api && \
./script/create_superuser_token.rb $(cat /tmp/superuser_token) && \
chown www-data:www-data config.ru && \
chown www-data:www-data log -R && \
+ mkdir -p tmp && \
chown www-data:www-data tmp -R
# Configure Apache and Passenger.
# config.compute_node_nameservers = ['1.2.3.4', '1.2.3.5']
require 'net/http'
config.compute_node_nameservers = [ '@@ARVADOS_DNS_SERVER@@' ]
-
+ config.compute_node_domain = false
config.uuid_prefix = '@@API_HOSTNAME@@'
# Authentication stub: hard code pre-approved API tokens.
fi
if [[ "$2" != '' ]]; then
local name="$2"
- args="$args -name $name"
+ args="$args --name $name"
fi
if [[ "$3" != '' ]]; then
local volume="$3"
fi
if [[ "$4" != '' ]]; then
local link="$4"
- args="$args -link $link"
+ args="$args --link $link"
fi
local image=$5
# Install prerequisite packages for Arvados
# * git, curl, rvm
-# * Arvados source code in /usr/src/arvados-upstream, for preseeding gem installation
+# * Arvados source code in /usr/src/arvados, for preseeding gem installation
RUN apt-get update && \
- apt-get -q -y install -q -y openssh-server apt-utils git curl locales postgresql-server-dev-9.1 && \
+ apt-get -q -y install -q -y openssh-server apt-utils git curl \
+ libcurl3 libcurl3-gnutls libcurl4-openssl-dev locales \
+ postgresql-server-dev-9.1 && \
/bin/mkdir -p /root/.ssh && \
/bin/sed -ri 's/# en_US.UTF-8 UTF-8/en_US.UTF-8 UTF-8/' /etc/locale.gen && \
/usr/sbin/locale-gen && \
- curl -L https://get.rvm.io | bash -s stable --ruby=2.1.0 && \
- git clone https://github.com/curoverse/arvados.git /usr/src/arvados-upstream
+ curl -L https://get.rvm.io | bash -s stable && \
+ /usr/local/rvm/bin/rvm install 2.1.0 && \
+ /bin/mkdir -p /usr/src/arvados
+
+ADD generated/arvados.tar.gz /usr/src/arvados/
# Set up RVM environment. These are just the env variables created by
# /usr/local/rvm/scripts/rvm, which can't be run from a non-login shell.
# https://github.com/rubygems/rubygems.org/issues/613.
RUN gem update --system && \
gem install bundler && \
- bundle install --gemfile=/usr/src/arvados-upstream/apps/workbench/Gemfile && \
- bundle install --gemfile=/usr/src/arvados-upstream/services/api/Gemfile && \
- bundle install --gemfile=/usr/src/arvados-upstream/doc/Gemfile
+ bundle install --gemfile=/usr/src/arvados/apps/workbench/Gemfile && \
+ bundle install --gemfile=/usr/src/arvados/services/api/Gemfile && \
+ bundle install --gemfile=/usr/src/arvados/doc/Gemfile
ADD generated/id_rsa.pub /root/.ssh/authorized_keys
RUN chown root:root /root/.ssh/authorized_keys
#! /bin/bash
-build_ok=true
-
-# Check that:
-# * IP forwarding is enabled in the kernel.
-
-if [ "$(/sbin/sysctl --values net.ipv4.ip_forward)" != "1" ]
-then
- echo >&2 "WARNING: IP forwarding must be enabled in the kernel."
- echo >&2 "Try: sudo sysctl net.ipv4.ip_forward=1"
- build_ok=false
-fi
-
-# * Docker can be found in the user's path
-# * The user is in the docker group
-# * cgroup is mounted
-# * the docker daemon is running
-
-if ! docker images > /dev/null 2>&1
+# make sure Ruby 1.9.3 is installed before proceeding
+if ! ruby -e 'exit RUBY_VERSION >= "1.9.3"' 2>/dev/null
then
- echo >&2 "WARNING: docker could not be run."
- echo >&2 "Please make sure that:"
- echo >&2 " * You have permission to read and write /var/run/docker.sock"
- echo >&2 " * a 'cgroup' volume is mounted on your machine"
- echo >&2 " * the docker daemon is running"
- build_ok=false
-fi
+ echo "Installing Arvados requires at least Ruby 1.9.3."
+ echo "You may need to enter your password."
+ read -p "Press Ctrl-C to abort, or else press ENTER to install ruby1.9.3 and continue. " unused
-# * config.yml exists
-if [ '!' -f config.yml ]
-then
- echo >&2 "WARNING: no config.yml found in the current directory"
- echo >&2 "Copy config.yml.example to config.yml and update it with settings for your site."
- build_ok=false
+ sudo apt-get update
+ sudo apt-get -y install ruby1.9.3
fi
-# If ok to build, then go ahead and run make
-if $build_ok
-then
- make $*
-fi
+build_tools/build.rb $*
# `make clean' removes the files generated in the build directory
# but does not remove any docker images generated in previous builds
clean:
+ -rm -rf build
-rm *-image */generated/*
-@rmdir */generated
# Dependencies for */generated files which are prerequisites
# for building docker images.
+CONFIG_RB = build_tools/config.rb
+
+BUILD = build/.buildstamp
+
BASE_DEPS = base/Dockerfile $(BASE_GENERATED)
API_DEPS = api/Dockerfile $(API_GENERATED)
SSO_DEPS = sso/passenger.conf $(SSO_GENERATED)
-BASE_GENERATED = base/generated
+BASE_GENERATED = base/generated/arvados.tar.gz
API_GENERATED = \
api/generated/apache2_vhost \
sso/seeds.rb.in \
sso/secret_token.rb.in
-$(BASE_GENERATED): config.yml
- ./config.rb
+$(BUILD):
+ mkdir -p build
+ rsync -rlp --exclude=docker/ --exclude='**/log/*' --exclude='**/tmp/*' \
+ --chmod=Da+rx,Fa+rX ../ build/
+ touch build/.buildstamp
+
+$(BASE_GENERATED): config.yml $(BUILD)
+ $(CONFIG_RB)
+ mkdir -p base/generated
+ tar -czf base/generated/arvados.tar.gz -C build .
$(API_GENERATED): config.yml $(API_GENERATED_IN)
- ./config.rb
+ $(CONFIG_RB)
$(WORKBENCH_GENERATED): config.yml $(WORKBENCH_GENERATED_IN)
- ./config.rb
+ $(CONFIG_RB)
$(WAREHOUSE_GENERATED): config.yml $(WAREHOUSE_GENERATED_IN)
- ./config.rb
+ $(CONFIG_RB)
$(SSO_GENERATED): config.yml $(SSO_GENERATED_IN)
- ./config.rb
+ $(CONFIG_RB)
# The docker build -q option suppresses verbose build output.
# Necessary to prevent failure on building warehouse; see
# ============================================================
# The main Arvados servers: api, doc, workbench, warehouse
-api-image: passenger-image $(API_DEPS)
+api-image: passenger-image $(BUILD) $(API_DEPS)
mkdir -p api/generated
- tar -c -z -f api/generated/api.tar.gz -C ../services api
+ tar -czf api/generated/api.tar.gz -C build/services api
$(DOCKER_BUILD) -t arvados/api api
- echo -n "Built at $(date)" > api-image
+ date >api-image
-doc-image: base-image $(DOC_DEPS)
+doc-image: base-image $(BUILD) $(DOC_DEPS)
mkdir -p doc/generated
- tar -c -z -f doc/generated/doc.tar.gz -C .. doc
+ tar -czf doc/generated/doc.tar.gz -C build doc
$(DOCKER_BUILD) -t arvados/doc doc
- echo -n "Built at $(date)" > doc-image
+ date >doc-image
-workbench-image: passenger-image $(WORKBENCH_DEPS)
+workbench-image: passenger-image $(BUILD) $(WORKBENCH_DEPS)
mkdir -p workbench/generated
- tar -c -z -f workbench/generated/workbench.tar.gz -C ../apps workbench
+ tar -czf workbench/generated/workbench.tar.gz -C build/apps workbench
$(DOCKER_BUILD) -t arvados/workbench workbench
- echo -n "Built at $(date)" > workbench-image
+ date >workbench-image
warehouse-image: base-image $(WAREHOUSE_DEPS)
$(DOCKER_BUILD) -t arvados/warehouse warehouse
- echo -n "Built at $(date)" > warehouse-image
+ date >warehouse-image
sso-image: passenger-image $(SSO_DEPS)
$(DOCKER_BUILD) -t arvados/sso sso
- echo -n "Built at $(date)" > sso-image
+ date >sso-image
# ============================================================
# The arvados/base image is the base Debian image plus packages
passenger-image: base-image
$(DOCKER_BUILD) -t arvados/passenger passenger
- echo -n "Built at $(date)" > passenger-image
+ date >passenger-image
base-image: debian-image $(BASE_DEPS)
$(DOCKER_BUILD) -t arvados/base base
- echo -n "Built at $(date)" > base-image
+ date >base-image
debian-image:
./mkimage-debootstrap.sh arvados/debian wheezy ftp://ftp.us.debian.org/debian/
- echo -n "Built at $(date)" > debian-image
-
+ date >debian-image
--- /dev/null
+#! /usr/bin/env ruby
+
+require 'optparse'
+require 'tempfile'
+require 'yaml'
+
+def main options
+ if not ip_forwarding_enabled?
+ warn "NOTE: IP forwarding must be enabled in the kernel."
+ warn "Turning IP forwarding on now."
+ sudo %w(/sbin/sysctl net.ipv4.ip_forward=1)
+ end
+
+ # Check that:
+ # * Docker is installed and can be found in the user's path
+ # * Docker can be run as a non-root user
+ # - TODO: put the user is in the docker group if necessary
+ # - TODO: mount cgroup automatically
+ # - TODO: start the docker service if not started
+
+ docker_path = %x(which docker).chomp
+ if docker_path.empty?
+ warn "Docker not found."
+ warn ""
+ warn "Please make sure that Docker has been installed and"
+ warn "can be found in your PATH."
+ warn ""
+ warn "Installation instructions for a variety of platforms can be found at"
+ warn "http://docs.docker.io/en/latest/installation/"
+ exit
+ elsif not docker_ok?
+ warn "WARNING: docker could not be run."
+ warn "Please make sure that:"
+ warn " * You have permission to read and write /var/run/docker.sock"
+ warn " * a 'cgroup' volume is mounted on your machine"
+ warn " * the docker daemon is running"
+ exit
+ end
+
+ # Check that debootstrap is installed.
+ if not debootstrap_ok?
+ warn "Installing debootstrap."
+ sudo '/usr/bin/apt-get', 'install', 'debootstrap'
+ end
+
+ # Generate a config.yml if it does not exist or is empty
+ if not File.size? 'config.yml'
+ print "Generating config.yml.\n"
+ print "Arvados needs to know the email address of the administrative user,\n"
+ print "so that when that user logs in they are automatically made an admin.\n"
+ print "This should be the email address you use to log in to Google.\n"
+ print "\n"
+ admin_email_address = ""
+ until is_valid_email? admin_email_address
+ print "Enter your Google ID email address here: "
+ admin_email_address = gets.strip
+ if not is_valid_email? admin_email_address
+ print "That doesn't look like a valid email address. Please try again.\n"
+ end
+ end
+
+ File.open 'config.yml', 'w' do |config_out|
+ config = YAML.load_file 'config.yml.example'
+ config['API_AUTO_ADMIN_USER'] = admin_email_address
+ config['API_HOSTNAME'] = generate_api_hostname
+ config['PUBLIC_KEY_PATH'] = find_or_create_ssh_key(config['API_HOSTNAME'])
+ config.each_key do |var|
+ if var.end_with?('_PW') or var.end_with?('_SECRET')
+ config[var] = rand(2**256).to_s(36)
+ end
+ config_out.write "#{var}: #{config[var]}\n"
+ end
+ end
+ end
+
+ # If all prerequisites are met, go ahead and build.
+ if ip_forwarding_enabled? and
+ docker_ok? and
+ debootstrap_ok? and
+ File.exists? 'config.yml'
+ warn "Building Arvados."
+ system '/usr/bin/make', '-f', options[:makefile], *ARGV
+ end
+end
+
+# sudo
+# Execute the arg list 'cmd' under sudo.
+# cmd can be passed either as a series of arguments or as a
+# single argument consisting of a list, e.g.:
+# sudo 'apt-get', 'update'
+# sudo(['/usr/bin/gpasswd', '-a', ENV['USER'], 'docker'])
+# sudo %w(/usr/bin/apt-get install lxc-docker)
+#
+def sudo(*cmd)
+ # user can pass a single list in as an argument
+ # to allow usage like: sudo %w(apt-get install foo)
+ warn "You may need to enter your password here."
+ if cmd.length == 1 and cmd[0].class == Array
+ cmd = cmd[0]
+ end
+ system '/usr/bin/sudo', *cmd
+end
+
+# is_valid_email?
+# Returns true if its arg looks like a valid email address.
+# This is a very very loose sanity check.
+#
+def is_valid_email? str
+ str.match /^\S+@\S+\.\S+$/
+end
+
+# generate_api_hostname
+# Generates a 5-character randomly chosen API hostname.
+#
+def generate_api_hostname
+ rand(2**256).to_s(36)[0...5]
+end
+
+# ip_forwarding_enabled?
+# Returns 'true' if IP forwarding is enabled in the kernel
+#
+def ip_forwarding_enabled?
+ %x(/sbin/sysctl -n net.ipv4.ip_forward) == "1\n"
+end
+
+# debootstrap_ok?
+# Returns 'true' if debootstrap is installed and working.
+#
+def debootstrap_ok?
+ return system '/usr/sbin/debootstrap --version > /dev/null 2>&1'
+end
+
+# docker_ok?
+# Returns 'true' if docker can be run as the current user.
+#
+def docker_ok?
+ return system 'docker images > /dev/null 2>&1'
+end
+
+# find_or_create_ssh_key arvados_name
+# Returns the SSH public key appropriate for this Arvados instance,
+# generating one if necessary.
+#
+def find_or_create_ssh_key arvados_name
+ ssh_key_file = "#{ENV['HOME']}/.ssh/arvados_#{arvados_name}_id_rsa"
+ unless File.exists? ssh_key_file
+ system 'ssh-keygen',
+ '-f', ssh_key_file,
+ '-C', "arvados@#{arvados_name}",
+ '-P', ''
+ end
+
+ return "#{ssh_key_file}.pub"
+end
+
+# install_docker
+# Determines which Docker package is suitable for this Linux distro
+# and installs it, resolving any dependencies.
+# NOTE: not in use yet.
+
+def install_docker
+ linux_distro = %x(lsb_release --id).split.last
+ linux_release = %x(lsb_release --release).split.last
+ linux_version = linux_distro + " " + linux_release
+ kernel_release = `uname -r`
+
+ case linux_distro
+ when 'Ubuntu'
+ if not linux_release.match '^1[234]\.'
+ warn "Arvados requires at least Ubuntu 12.04 (Precise Pangolin)."
+ warn "Your system is Ubuntu #{linux_release}."
+ exit
+ end
+ if linux_release.match '^12' and kernel_release.start_with? '3.2'
+ # Ubuntu Precise ships with a 3.2 kernel and must be upgraded.
+ warn "Your kernel #{kernel_release} must be upgraded to run Docker."
+ warn "To do this:"
+ warn " sudo apt-get update"
+ warn " sudo apt-get install linux-image-generic-lts-raring linux-headers-generic-lts-raring"
+ warn " sudo reboot"
+ exit
+ else
+ # install AUFS
+ sudo 'apt-get', 'update'
+ sudo 'apt-get', 'install', "linux-image-extra-#{kernel_release}"
+ end
+
+ # add Docker repository
+ sudo %w(/usr/bin/apt-key adv
+ --keyserver keyserver.ubuntu.com
+ --recv-keys 36A1D7869245C8950F966E92D8576A8BA88D21E9)
+ source_file = Tempfile.new('arv')
+ source_file.write("deb http://get.docker.io/ubuntu docker main\n")
+ source_file.close
+ sudo '/bin/mv', source_file.path, '/etc/apt/sources.list.d/docker.list'
+ sudo %w(/usr/bin/apt-get update)
+ sudo %w(/usr/bin/apt-get install lxc-docker)
+
+ # Set up for non-root access
+ sudo %w(/usr/sbin/groupadd docker)
+ sudo '/usr/bin/gpasswd', '-a', ENV['USER'], 'docker'
+ sudo %w(/usr/sbin/service docker restart)
+ when 'Debian'
+ else
+ warn "Must be running a Debian or Ubuntu release in order to run Docker."
+ exit
+ end
+end
+
+
+if __FILE__ == $PROGRAM_NAME
+ options = { :makefile => File.join(File.dirname(__FILE__), 'Makefile') }
+ OptionParser.new do |opts|
+ opts.on('-m', '--makefile MAKEFILE-PATH',
+ 'Path to the Makefile used to build Arvados Docker images') do |mk|
+ options[:makefile] = mk
+ end
+ end
+
+ main options
+end
# For each *.in file in the docker directories, substitute any
# @@variables@@ found in the file with the appropriate config
# variable. Support up to 10 levels of nesting.
-#
+#
# TODO(twp): add the *.in files directory to the source tree, and
# when expanding them, add them to the "generated" directory with
# the same tree structure as in the original source. Then all
File.delete(stale_file)
end
+File.umask(022)
Dir.glob('*/*.in') do |template_file|
generated_dir = File.join(File.dirname(template_file), 'generated')
Dir.mkdir(generated_dir) unless Dir.exists? generated_dir
output_path = File.join(generated_dir, File.basename(template_file, '.in'))
- output = File.open(output_path, "w")
- File.open(template_file) do |input|
- input.each_line do |line|
+ File.open(output_path, "w") do |output|
+ File.open(template_file) do |input|
+ input.each_line do |line|
- @count = 0
- while @count < 10
- @out = line.gsub!(/@@(.*?)@@/) do |var|
- if config.key?(Regexp.last_match[1])
- config[Regexp.last_match[1]]
- else
- var.gsub!(/@@/, '@_NOT_FOUND_@')
+ # This count is used to short-circuit potential
+ # infinite loops of variable substitution.
+ @count = 0
+ while @count < 10
+ @out = line.gsub!(/@@(.*?)@@/) do |var|
+ if config.key?(Regexp.last_match[1])
+ config[Regexp.last_match[1]]
+ else
+ var.gsub!(/@@/, '@_NOT_FOUND_@')
+ end
end
+ break if @out.nil?
+ @count += 1
end
- break if @out.nil?
- @count += 1
- end
- output.write(line)
+ output.write(line)
+ end
end
end
- output.close
end
# Copy the ssh public key file to base/generated (if a path is given)
generated_dir = File.join('base/generated')
Dir.mkdir(generated_dir) unless Dir.exists? generated_dir
-if config.key?('PUBLIC_KEY_PATH') &&
- ! (config['PUBLIC_KEY_PATH'] == '') &&
- File.readable?(config['PUBLIC_KEY_PATH'])
+if (!config['PUBLIC_KEY_PATH'].nil? and
+ File.readable? config['PUBLIC_KEY_PATH'])
FileUtils.cp(config['PUBLIC_KEY_PATH'],
File.join(generated_dir, 'id_rsa.pub'))
end
ADD generated/doc.tar.gz /usr/src/arvados/
# Build static site
-RUN /bin/sed -ri 's/^baseurl: .*$/baseurl: /' /usr/src/arvados/doc/_config.yml && \
+RUN bundle install --gemfile=/usr/src/arvados/doc/Gemfile && \
+ /bin/sed -ri 's/^baseurl: .*$/baseurl: /' /usr/src/arvados/doc/_config.yml && \
cd /usr/src/arvados/doc && \
LANG="en_US.UTF-8" LC_ALL="en_US.UTF-8" rake
+++ /dev/null
-#! /bin/bash
-
-# Wrapper script for `docker build'.
-# This is a workaround for https://github.com/dotcloud/docker/issues/1875.
-
-tmpfile=$(mktemp)
-trap "rm $tmpfile; exit 1" SIGHUP SIGINT SIGTERM
-
-docker build $* | tee ${tmpfile}
-if $(grep -q 'Error build' ${tmpfile})
-then
- result=1
-else
- result=0
-fi
-
-rm $tmpfile
-exit $result
--- /dev/null
+#! /bin/sh
+
+# Install prerequisites.
+sudo apt-get install curl libcurl3 libcurl3-gnutls libcurl4-openssl-dev python-pip
+
+# Install RVM.
+curl -sSL https://get.rvm.io | bash -s stable
+source ~/.rvm/scripts/rvm
+rvm install 2.1.0
+
+# Install arvados-cli.
+gem install arvados-cli
+sudo pip install --upgrade httplib2
# Update Arvados source
RUN /bin/mkdir -p /usr/src/arvados/apps
ADD generated/workbench.tar.gz /usr/src/arvados/apps/
+ADD generated/secret_token.rb /usr/src/arvados/apps/workbench/config/initializers/secret_token.rb
+ADD generated/production.rb /usr/src/arvados/apps/workbench/config/environments/production.rb
+ADD passenger.conf /etc/apache2/conf.d/passenger
+
-RUN touch /usr/src/arvados/apps/workbench/log/production.log && \
+RUN bundle install --gemfile=/usr/src/arvados/apps/workbench/Gemfile && \
+ touch /usr/src/arvados/apps/workbench/log/production.log && \
chmod 666 /usr/src/arvados/apps/workbench/log/production.log && \
touch /usr/src/arvados/apps/workbench/db/production.sqlite3 && \
bundle install --gemfile=/usr/src/arvados/apps/workbench/Gemfile && \
cd /usr/src/arvados/apps/workbench && \
- rake assets:precompile
+ rake assets:precompile && \
+ chown -R www-data:www-data /usr/src/arvados/apps/workbench
# Configure Apache
ADD generated/apache2_vhost /etc/apache2/sites-available/workbench
a2ensite workbench && \
a2enmod rewrite
-# Set up the production environment
-ADD generated/secret_token.rb /usr/src/arvados/apps/workbench/config/initializers/secret_token.rb
-ADD generated/production.rb /usr/src/arvados/apps/workbench/config/environments/production.rb
-ADD passenger.conf /etc/apache2/conf.d/passenger
-
ADD apache2_foreground.sh /etc/apache2/foreground.sh
# Start Apache
s.executables << "arv-run-pipeline-instance"
s.executables << "arv-crunch-job"
s.executables << "arv-tag"
+ s.add_runtime_dependency 'arvados', '~> 0.1.0'
s.add_runtime_dependency 'google-api-client', '~> 0.6.3'
s.add_runtime_dependency 'activesupport', '~> 3.2', '>= 3.2.13'
s.add_runtime_dependency 'json', '~> 1.7', '>= 1.7.7'
rescue LoadError
abort <<-EOS
-Please install all required gems:
+Please install all required gems:
gem install activesupport andand curb google-api-client json oj trollop
end
end
-client = ArvadosClient.new(:host => ENV['ARVADOS_API_HOST'], :application_name => 'arvados-cli', :application_version => '1.0')
-arvados = client.discovered_api('arvados', ENV['ARVADOS_API_VERSION'])
+begin
+ client = ArvadosClient.new(:host => ENV['ARVADOS_API_HOST'], :application_name => 'arvados-cli', :application_version => '1.0')
+ arvados = client.discovered_api('arvados', ENV['ARVADOS_API_VERSION'])
+rescue Exception => e
+ puts "Failed to connect to Arvados API server: #{e}"
+ exit 1
+end
def to_boolean(s)
!!(s =~ /^(true|t|yes|y|1)$/i)
end
banner += "\n"
STDERR.puts banner
-
- if not method.nil? and method != '--help' then
+
+ if not method.nil? and method != '--help' then
Trollop::die ("Unknown method #{method.inspect} " +
"for resource #{resource.inspect}")
end
banner += "\n"
STDERR.puts banner
- if not resource.nil? and resource != '--help' then
+ if not resource.nil? and resource != '--help' then
Trollop::die "Unknown resource type #{resource.inspect}"
end
exit 255
curl.headers['Accept'] = 'text/plain'
curl.headers['Authorization'] = "OAuth2 #{ENV['ARVADOS_API_TOKEN']}"
if ENV['ARVADOS_API_HOST_INSECURE']
- curl.ssl_verify_peer = false
+ curl.ssl_verify_peer = false
curl.ssl_verify_host = false
end
if global_opts[:verbose]
puts results['uuid']
end
end
-
-
# == Synopsis
#
-# wh-run-pipeline-instance --template pipeline-template-uuid [options] [--] [parameters]
-# wh-run-pipeline-instance --instance pipeline-instance-uuid [options]
+# arv-run-pipeline-instance --template pipeline-template-uuid [options] [--] [parameters]
+# arv-run-pipeline-instance --instance pipeline-instance-uuid [options]
#
# Satisfy a pipeline template by finding or submitting a mapreduce job
# for each pipeline component.
# to finish. Just find out whether jobs are finished,
# queued, or running for each component
#
-# [--create-instance-only] Do not try to satisfy any components. Just
+# [--submit] Do not try to satisfy any components. Just
# create an instance, print its UUID to
# stdout, and exit.
#
# [--no-wait] Make only as much progress as possible without entering
# a sleep/poll loop.
#
-# [--no-reuse-finished] Do not reuse existing outputs to satisfy
-# pipeline components. Always submit a new job
-# or use an existing job which has not yet
-# finished.
-#
# [--no-reuse] Do not reuse existing jobs to satisfy pipeline
# components. Submit a new job for every component.
#
abort "#{$0}: fatal: ARVADOS_API_TOKEN environment variable not set."
begin
+ require 'arvados'
require 'rubygems'
- require 'google/api_client'
require 'json'
require 'pp'
require 'trollop'
-rescue LoadError
+ require 'google/api_client'
+rescue LoadError => l
+ puts $:
abort <<-EOS
-#{$0}: fatal: some runtime dependencies are missing.
-Try: gem install pp google-api-client json trollop
+#{$0}: fatal: #{l.message}
+Some runtime dependencies may be missing.
+Try: gem install arvados pp google-api-client json trollop
EOS
end
"Do not wait for jobs to finish. Just look up status, submit new jobs if needed, and exit.",
:short => :none,
:type => :boolean)
- opt(:no_reuse_finished,
- "Do not reuse existing outputs to satisfy pipeline components. Always submit a new job or use an existing job which has not yet finished.",
- :short => :none,
- :type => :boolean)
opt(:no_reuse,
"Do not reuse existing jobs to satisfy pipeline components. Submit a new job for every component.",
:short => :none,
"UUID of pipeline instance.",
:short => :none,
:type => :string)
- opt(:create_instance_only,
+ opt(:submit,
"Do not try to satisfy any components. Just create a pipeline instance and output its UUID.",
:short => :none,
:type => :boolean)
+ opt(:run_here,
+ "Manage the pipeline in process.",
+ :short => :none,
+ :type => :boolean)
stop_on [:'--']
end
$options = Trollop::with_standard_exception_handling p do
$debuglevel = $options[:debug_level] || ($options[:debug] && 1) || 0
if $options[:instance]
- if $options[:template] or $options[:create_instance_only]
- abort "#{$0}: syntax error: --instance cannot be combined with --template or --create-instance-only."
+ if $options[:template] or $options[:submit]
+ abort "#{$0}: syntax error: --instance cannot be combined with --template or --submit."
end
elsif not $options[:template]
abort "#{$0}: syntax error: you must supply a --template or --instance."
end
+if $options[:run_here] == $options[:submit]
+ abort "#{$0}: syntax error: you must supply either --run-here or --submit."
+end
+
+# Suppress SSL certificate checks if ARVADOS_API_HOST_INSECURE
+
+module Kernel
+ def suppress_warnings
+ original_verbosity = $VERBOSE
+ $VERBOSE = nil
+ result = yield
+ $VERBOSE = original_verbosity
+ return result
+ end
+end
+
+if ENV['ARVADOS_API_HOST_INSECURE']
+ suppress_warnings { OpenSSL::SSL::VERIFY_PEER = OpenSSL::SSL::VERIFY_NONE }
+end
+
# Set up the API client.
$client ||= Google::APIClient.
:application_name => File.split($0).last,
:application_version => $application_version.to_s)
$arvados = $client.discovered_api('arvados', $arvados_api_version)
+$arv = Arvados.new api_version: 'v1'
class PipelineInstance
if j.is_a? Hash and j[:uuid]
@cache[j[:uuid]] = j
else
- debuglog "create job: #{j[:errors] rescue nil}", 0
+ debuglog "create job: #{j[:errors] rescue nil} with attribute #{attributes}", 0
nil
end
end
moretodo = false
@components.each do |cname, c|
job = nil
+ # Is the job satisfying this component already known to be
+ # finished? (Already meaning "before we query API server about
+ # the job's current state")
+ c_already_finished = (c[:job] &&
+ c[:job][:uuid] &&
+ !c[:job][:success].nil?)
if !c[:job] and
- c[:script_parameters].select { |pname, p| p.is_a? Hash }.empty?
- # Job is fully specified (all parameter values are present) but
- # no particular job has been found.
-
- debuglog "component #{cname} ready to satisfy."
-
- c.delete :wait
- second_place_job = nil # satisfies component, but not finished yet
-
- (@options[:no_reuse] ? [] : JobCache.
- where(script: c[:script],
- script_parameters: c[:script_parameters],
- script_version_descends_from: c[:script_version])
- ).each do |candidate_job|
- candidate_params_downcase = Hash[candidate_job[:script_parameters].
- map { |k,v| [k.downcase,v] }]
- c_params_downcase = Hash[c[:script_parameters].
- map { |k,v| [k.downcase,v] }]
-
- debuglog "component #{cname} considering job #{candidate_job[:uuid]} version #{candidate_job[:script_version]} parameters #{candidate_params_downcase.inspect}", 3
-
- unless candidate_params_downcase == c_params_downcase
- next
- end
-
- if c[:script_version] !=
- candidate_job[:script_version][0,c[:script_version].length]
- debuglog "component #{cname} would be satisfied by job #{candidate_job[:uuid]} if script_version matched.", 2
- next
- end
-
- unless candidate_job[:success] || candidate_job[:running] ||
- (!candidate_job[:started_at] && !candidate_job[:cancelled_at])
- debuglog "component #{cname} would be satisfied by job #{candidate_job[:uuid]} if it were running or successful.", 2
- next
- end
-
- if candidate_job[:success]
- unless @options[:no_reuse_finished]
- job = candidate_job
- $stderr.puts "using #{job[:uuid]} (finished at #{job[:finished_at]}) for component #{cname}"
- c[:job] = job
- end
- else
- second_place_job ||= candidate_job
- end
- break
- end
- if not c[:job] and second_place_job
- job = second_place_job
- $stderr.puts "using #{job[:uuid]} (running since #{job[:started_at]}) for component #{cname}"
+ c[:script_parameters].select { |pname, p| p.is_a? Hash and p[:output_of]}.empty?
+ # No job yet associated with this component and is component inputs
+ # are fully specified (any output_of script_parameters are resolved
+ # to real value)
+ job = JobCache.create({:script => c[:script],
+ :script_parameters => c[:script_parameters],
+ :script_version => c[:script_version],
+ :repository => c[:repository],
+ :minimum_script_version => c[:minimum_script_version],
+ :exclude_script_versions => c[:exclude_minimum_script_versions],
+ :nondeterministic => c[:nondeterministic],
+ :no_reuse => @options[:no_reuse],
+ :output_is_persistent => c[:output_is_persistent] || false})
+ if job
+ debuglog "component #{cname} new job #{job[:uuid]}"
c[:job] = job
+ else
+ debuglog "component #{cname} new job failed"
end
- if not c[:job]
- debuglog "component #{cname} not satisfied by any existing job."
- if !@options[:dry_run]
- debuglog "component #{cname} new job."
- job = JobCache.create(:script => c[:script],
- :script_parameters => c[:script_parameters],
- :runtime_constraints => c[:runtime_constraints] || {},
- :script_version => c[:script_version] || 'master')
- if job
- debuglog "component #{cname} new job #{job[:uuid]}"
- c[:job] = job
- else
- debuglog "component #{cname} new job failed"
- end
- end
- end
- else
- c[:wait] = true
end
+
if c[:job] and c[:job][:uuid]
if (c[:job][:running] or
not (c[:job][:finished_at] or c[:job][:cancelled_at]))
+ # Job is running so update copy of job record
c[:job] = JobCache.get(c[:job][:uuid])
end
+
if c[:job][:success]
# Populate script_parameters of other components waiting for
# this job
if p.is_a? Hash and p[:output_of] == cname.to_s
debuglog "parameter #{c2name}::#{pname} == #{c[:job][:output]}"
c2[:script_parameters][pname] = c[:job][:output]
+ moretodo = true
+ end
+ end
+ end
+ unless c_already_finished
+ # This is my first time discovering that the job
+ # succeeded. (At the top of this loop, I was still
+ # waiting for it to finish.)
+ if c[:output_is_persistent]
+ # I need to make sure a resources/wants link is in
+ # place to protect the output from garbage
+ # collection. (Normally Crunch does this for me, but
+ # here I might be reusing the output of someone else's
+ # job and I need to make sure it's understood that the
+ # output is valuable to me, too.)
+ wanted = c[:job][:output]
+ debuglog "checking for existing persistence link for #{wanted}"
+ @my_user_uuid ||= $arv.user.current[:uuid]
+ links = $arv.link.list(limit: 1,
+ filters:
+ [%w(link_class = resources),
+ %w(name = wants),
+ %w(tail_uuid =) + [@my_user_uuid],
+ %w(head_uuid =) + [wanted]
+ ])[:items]
+ if links.any?
+ debuglog "link already exists, uuid #{links.first[:uuid]}"
+ else
+ newlink = $arv.link.create link: \
+ {
+ link_class: 'resources',
+ name: 'wants',
+ tail_kind: 'arvados#user',
+ tail_uuid: @my_user_uuid,
+ head_kind: 'arvados#collection',
+ head_uuid: wanted
+ }
+ debuglog "added link, uuid #{newlink[:uuid]}"
end
end
end
elsif c[:job][:running] ||
(!c[:job][:started_at] && !c[:job][:cancelled_at])
- moretodo ||= !@options[:no_wait]
+ # Job is still running
+ moretodo = true
elsif c[:job][:cancelled_at]
debuglog "component #{cname} job #{c[:job][:uuid]} cancelled."
end
@instance[:components] = @components
@instance[:active] = moretodo
report_status
+
+ if @options[:no_wait]
+ moretodo = false
+ end
+
if moretodo
begin
sleep 10
end
end
end
- @instance[:success] = @components.reject { |cname,c| c[:job] and c[:job][:success] }.empty?
+
+ ended = 0
+ succeeded = 0
+ failed = 0
+ @components.each do |cname, c|
+ if c[:job]
+ if c[:job][:finished_at]
+ ended += 1
+ if c[:job][:success] == true
+ succeeded += 1
+ elsif c[:job][:success] == false
+ failed += 1
+ end
+ end
+ end
+ end
+
+ if ended == @components.length or failed > 0
+ @instance[:active] = false
+ @instance[:success] = (succeeded == @components.length)
+ end
+
@instance.save
end
end
runner.apply_parameters(p.leftovers)
runner.setup_instance
- if $options[:create_instance_only]
+ if $options[:submit]
runner.instance.save
puts runner.instance[:uuid]
else
#!/usr/bin/perl
-# -*- mode: perl; perl-indent-level: 2; -*-
+# -*- mode: perl; perl-indent-level: 2; indent-tabs-mode: nil; -*-
=head1 NAME
Arvados API authorization token to use during the course of the job.
+=item --no-clear-tmp
+
+Do not clear per-job/task temporary directories during initial job
+setup. This can speed up development and debugging when running jobs
+locally.
+
=back
=head1 RUNNING JOBS LOCALLY
use Fcntl qw(F_GETFL F_SETFL O_NONBLOCK);
use Arvados;
use Getopt::Long;
-use Warehouse;
-use Warehouse::Stream;
-use IPC::System::Simple qw(capturex);
+use IPC::Open2;
+use IO::Select;
+use File::Temp;
+use Fcntl ':flock';
$ENV{"TMPDIR"} ||= "/tmp";
unless (defined $ENV{"CRUNCH_TMP"}) {
my $git_dir;
my $jobspec;
my $job_api_token;
+my $no_clear_tmp;
my $resume_stash;
GetOptions('force-unlock' => \$force_unlock,
'git-dir=s' => \$git_dir,
'job=s' => \$jobspec,
'job-api-token=s' => \$job_api_token,
+ 'no-clear-tmp' => \$no_clear_tmp,
'resume-stash=s' => \$resume_stash,
);
}
$job_id = $Job->{'uuid'};
-$metastream = Warehouse::Stream->new(whc => new Warehouse);
-$metastream->clear;
-$metastream->name('.');
-$metastream->write_start($job_id . '.log.txt');
-
+my $keep_logfile = $job_id . '.log.txt';
+my $local_logfile = File::Temp->new();
$Job->{'runtime_constraints'} ||= {};
$Job->{'runtime_constraints'}->{'max_tasks_per_node'} ||= 0;
}
if (exists $ENV{SLURM_NODELIST})
{
- push @sinfo, `sinfo -h --format='%c %N' --nodes='$ENV{SLURM_NODELIST}'`;
+ push @sinfo, `sinfo -h --format='%c %N' --nodes=\Q$ENV{SLURM_NODELIST}\E`;
}
foreach (@sinfo)
{
}
+if (!$have_slurm)
+{
+ must_lock_now("$ENV{CRUNCH_TMP}/.lock", "a job is already running here.");
+}
+
+
my $build_script;
my $skip_install = ($local_job && $Job->{script_version} =~ m{^/});
if ($skip_install)
{
+ if (!defined $no_clear_tmp) {
+ my $clear_tmp_cmd = 'rm -rf $JOB_WORK $CRUNCH_TMP/opt $CRUNCH_TMP/src*';
+ system($clear_tmp_cmd) == 0
+ or croak ("`$clear_tmp_cmd` failed: ".($?>>8));
+ }
$ENV{"CRUNCH_SRC"} = $Job->{script_version};
for my $src_path ("$ENV{CRUNCH_SRC}/arvados/sdk/python") {
if (-d $src_path) {
Log (undef, "Install revision ".$Job->{script_version});
my $nodelist = join(",", @node);
- # Clean out crunch_tmp/work, crunch_tmp/opt, crunch_tmp/src*
+ if (!defined $no_clear_tmp) {
+ # Clean out crunch_tmp/work, crunch_tmp/opt, crunch_tmp/src*
- my $cleanpid = fork();
- if ($cleanpid == 0)
- {
- srun (["srun", "--nodelist=$nodelist", "-D", $ENV{'TMPDIR'}],
- ['bash', '-c', 'if mount | grep -q $JOB_WORK/; then sudo /bin/umount $JOB_WORK/* 2>/dev/null; fi; sleep 1; rm -rf $JOB_WORK $CRUNCH_TMP/opt $CRUNCH_TMP/src*']);
- exit (1);
- }
- while (1)
- {
- last if $cleanpid == waitpid (-1, WNOHANG);
- freeze_if_want_freeze ($cleanpid);
- select (undef, undef, undef, 0.1);
+ my $cleanpid = fork();
+ if ($cleanpid == 0)
+ {
+ srun (["srun", "--nodelist=$nodelist", "-D", $ENV{'TMPDIR'}],
+ ['bash', '-c', 'if mount | grep -q $JOB_WORK/; then sudo /bin/umount $JOB_WORK/* 2>/dev/null; fi; sleep 1; rm -rf $JOB_WORK $CRUNCH_TMP/opt $CRUNCH_TMP/src*']);
+ exit (1);
+ }
+ while (1)
+ {
+ last if $cleanpid == waitpid (-1, WNOHANG);
+ freeze_if_want_freeze ($cleanpid);
+ select (undef, undef, undef, 0.1);
+ }
+ Log (undef, "Clean-work-dir exited $?");
}
- Log (undef, "Clean-work-dir exited $?");
# Install requested code version
my $commit;
my $git_archive;
my $treeish = $Job->{'script_version'};
- my $repo = $git_dir || $ENV{'CRUNCH_DEFAULT_GIT_DIR'};
- # Todo: let script_version specify repository instead of expecting
- # parent process to figure it out.
- $ENV{"CRUNCH_SRC_URL"} = $repo;
- # Create/update our clone of the remote git repo
+ # If we're running under crunch-dispatch, it will have pulled the
+ # appropriate source tree into its own repository, and given us that
+ # repo's path as $git_dir. If we're running a "local" job, and a
+ # script_version was specified, it's up to the user to provide the
+ # full path to a local repository in Job->{repository}.
+ #
+ # TODO: Accept URLs too, not just local paths. Use git-ls-remote and
+ # git-archive --remote where appropriate.
+ #
+ # TODO: Accept a locally-hosted Arvados repository by name or
+ # UUID. Use arvados.v1.repositories.list or .get to figure out the
+ # appropriate fetch-url.
+ my $repo = $git_dir || $ENV{'CRUNCH_DEFAULT_GIT_DIR'} || $Job->{'repository'};
- if (!-d $ENV{"CRUNCH_SRC"}) {
- system(qw(git clone), $repo, $ENV{"CRUNCH_SRC"}) == 0
- or croak ("git clone $repo failed: exit ".($?>>8));
- system("cd $ENV{CRUNCH_SRC} && git config clean.requireForce false");
+ $ENV{"CRUNCH_SRC_URL"} = $repo;
+
+ if (-d "$repo/.git") {
+ # We were given a working directory, but we are only interested in
+ # the index.
+ $repo = "$repo/.git";
}
- `cd $ENV{CRUNCH_SRC} && git remote set-url origin \"\$CRUNCH_SRC_URL\" && git fetch -q origin`;
# If this looks like a subversion r#, look for it in git-svn commit messages
if ($treeish =~ m{^\d{1,4}$}) {
- my $gitlog = `cd $ENV{CRUNCH_SRC} && git log --pretty="format:%H" --grep="git-svn-id:.*\@$treeish " origin/master`;
+ my $gitlog = `git --git-dir=\Q$repo\E log --pretty="format:%H" --grep="git-svn-id:.*\@"\Q$treeish\E" " master`;
chomp $gitlog;
if ($gitlog =~ /^[a-f0-9]{40}$/) {
$commit = $gitlog;
# If that didn't work, try asking git to look it up as a tree-ish.
if (!defined $commit) {
-
- my $cooked_treeish = $treeish;
- if ($treeish !~ m{^[0-9a-f]{5,}$}) {
- # Looks like a git branch name -- make sure git knows it's
- # relative to the remote repo
- $cooked_treeish = "origin/$treeish";
- }
-
- my $found = `cd $ENV{CRUNCH_SRC} && git rev-list -1 $cooked_treeish`;
+ my $found = `git --git-dir=\Q$repo\E rev-list -1 ''\Q$treeish\E`;
chomp $found;
if ($found =~ /^[0-9a-f]{40}$/s) {
$commit = $found;
$ENV{"CRUNCH_SRC_COMMIT"} = $commit;
@execargs = ("sh", "-c",
"mkdir -p $ENV{CRUNCH_INSTALL} && cd $ENV{CRUNCH_TMP} && perl -");
- $git_archive = `cd $ENV{CRUNCH_SRC} && git archive $commit`;
+ $git_archive = `git --git-dir=\Q$repo\E archive ''\Q$commit\E`;
}
else {
croak ("could not figure out commit id for $treeish");
Log (undef, "Install exited $?");
}
+if (!$have_slurm)
+{
+ # Grab our lock again (we might have deleted and re-created CRUNCH_TMP above)
+ must_lock_now("$ENV{CRUNCH_TMP}/.lock", "a job is already running here.");
+}
+
foreach (qw (script script_version script_parameters runtime_constraints))
}
$ENV{"TASK_SLOT_NODE"} = $slot[$childslot]->{node}->{name};
$ENV{"TASK_SLOT_NUMBER"} = $slot[$childslot]->{cpu};
- $ENV{"TASK_WORK"} = $ENV{"JOB_WORK"}."/".$slot[$childslot]->{cpu};
- $ENV{"TASK_KEEPMOUNT"} = $ENV{"TASK_WORK"}."/keep";
+ $ENV{"TASK_WORK"} = $ENV{"JOB_WORK"}."/$id.$$";
+ $ENV{"TASK_KEEPMOUNT"} = $ENV{"TASK_WORK"}.".keep";
$ENV{"TASK_TMPDIR"} = $ENV{"TASK_WORK"}; # deprecated
$ENV{"CRUNCH_NODE_SLOTS"} = $slot[$childslot]->{node}->{ncpus};
$ENV{"PATH"} = $ENV{"CRUNCH_INSTALL"} . "/bin:" . $ENV{"PATH"};
$command .=
"&& perl -";
}
- $ENV{"PYTHONPATH"} =~ s{^}{:} if $ENV{"PYTHONPATH"};
- $ENV{"PYTHONPATH"} =~ s{^}{$ENV{CRUNCH_SRC}/sdk/python}; # xxx hack
- $ENV{"PYTHONPATH"} =~ s{^}{$ENV{CRUNCH_SRC}/arvados/sdk/python:}; # xxx hack
- $ENV{"PYTHONPATH"} =~ s{$}{:/usr/local/arvados/src/sdk/python}; # xxx hack
$command .=
"&& exec arv-mount $ENV{TASK_KEEPMOUNT} --exec $ENV{CRUNCH_SRC}/crunch_scripts/" . $Job->{"script"};
my @execargs = ('bash', '-c', $command);
if ($Job->{'output'})
{
eval {
- my $manifest_text = capturex("whget", $Job->{'output'});
+ my $manifest_text = `arv keep get ''\Q$Job->{'output'}\E`;
$arv->{'collections'}->{'create'}->execute('collection' => {
'uuid' => $Job->{'output'},
'manifest_text' => $manifest_text,
});
+ if ($Job->{'output_is_persistent'}) {
+ $arv->{'links'}->{'create'}->execute('link' => {
+ 'tail_kind' => 'arvados#user',
+ 'tail_uuid' => $User->{'uuid'},
+ 'head_kind' => 'arvados#collection',
+ 'head_uuid' => $Job->{'output'},
+ 'link_class' => 'resources',
+ 'name' => 'wants',
+ });
+ }
};
if ($@) {
Log (undef, "Failed to register output manifest: $@");
} split ("\n", $jobstep[$job]->{stderr});
}
+sub fetch_block
+{
+ my $hash = shift;
+ my ($keep, $child_out, $output_block);
+
+ my $cmd = "arv keep get \Q$hash\E";
+ open($keep, '-|', $cmd) or die "fetch_block: $cmd: $!";
+ sysread($keep, $output_block, 64 * 1024 * 1024);
+ close $keep;
+ return $output_block;
+}
sub collate_output
{
- my $whc = Warehouse->new;
Log (undef, "collate");
- $whc->write_start (1);
+
+ my ($child_out, $child_in);
+ my $pid = open2($child_out, $child_in, 'arv', 'keep', 'put', '--raw');
my $joboutput;
for (@jobstep)
{
if ($output !~ /^[0-9a-f]{32}(\+\S+)*$/)
{
$output_in_keep ||= $output =~ / [0-9a-f]{32}\S*\+K/;
- $whc->write_data ($output);
+ print $child_in $output;
}
elsif (@jobstep == 1)
{
$joboutput = $output;
- $whc->write_finish;
+ last;
}
- elsif (defined (my $outblock = $whc->fetch_block ($output)))
+ elsif (defined (my $outblock = fetch_block ($output)))
{
$output_in_keep ||= $outblock =~ / [0-9a-f]{32}\S*\+K/;
- $whc->write_data ($outblock);
+ print $child_in $outblock;
}
else
{
- my $errstr = $whc->errstr;
- $whc->write_data ("XXX fetch_block($output) failed: $errstr XXX\n");
+ Log (undef, "XXX fetch_block($output) failed XXX");
$main::success = 0;
}
}
- $joboutput = $whc->write_finish if !defined $joboutput;
+ $child_in->close;
+
+ if (!defined $joboutput) {
+ my $s = IO::Select->new($child_out);
+ if ($s->can_read(120)) {
+ sysread($child_out, $joboutput, 64 * 1024 * 1024);
+ chomp($joboutput);
+ } else {
+ Log (undef, "timed out reading from 'arv keep put'");
+ }
+ }
+ waitpid($pid, 0);
+
if ($joboutput)
{
Log (undef, "output $joboutput");
}
print STDERR ((-t STDERR) ? ($datetime." ".$message) : $message);
- return if !$metastream;
- $metastream->write_data ($datetime . " " . $message);
+ if ($metastream) {
+ print $metastream $datetime . " " . $message;
+ }
}
sub save_meta
{
my $justcheckpoint = shift; # false if this will be the last meta saved
- my $m = $metastream;
- $m = $m->copy if $justcheckpoint;
- $m->write_finish;
- my $whc = Warehouse->new;
- my $loglocator = $whc->store_block ($m->as_string);
- $arv->{'collections'}->{'create'}->execute('collection' => {
- 'uuid' => $loglocator,
- 'manifest_text' => $m->as_string,
- });
- undef $metastream if !$justcheckpoint; # otherwise Log() will try to use it
+ return if $justcheckpoint; # checkpointing is not relevant post-Warehouse.pm
+
+ $local_logfile->flush;
+ my $cmd = "arv keep put --filename ''\Q$keep_logfile\E "
+ . quotemeta($local_logfile->filename);
+ my $loglocator = `$cmd`;
+ die "system $cmd failed: $?" if $?;
+
+ $local_logfile = undef; # the temp file is automatically deleted
Log (undef, "log manifest is $loglocator");
$Job->{'log'} = $loglocator;
$Job->update_attributes('log', $loglocator) if $job_has_uuid;
sub thaw
{
croak ("Thaw not implemented");
-
- my $whc;
- my $key = shift;
- Log (undef, "thaw from $key");
-
- @jobstep = ();
- @jobstep_done = ();
- @jobstep_todo = ();
- @jobstep_tomerge = ();
- $jobstep_tomerge_level = 0;
- my $frozenjob = {};
-
- my $stream = new Warehouse::Stream ( whc => $whc,
- hash => [split (",", $key)] );
- $stream->rewind;
- while (my $dataref = $stream->read_until (undef, "\n\n"))
- {
- if ($$dataref =~ /^job /)
- {
- foreach (split ("\n", $$dataref))
- {
- my ($k, $v) = split ("=", $_, 2);
- $frozenjob->{$k} = freezeunquote ($v);
- }
- next;
- }
-
- if ($$dataref =~ /^merge (\d+) (.*)/)
- {
- $jobstep_tomerge_level = $1;
- @jobstep_tomerge
- = map { freezeunquote ($_) } split ("\n", freezeunquote($2));
- next;
- }
-
- my $Jobstep = { };
- foreach (split ("\n", $$dataref))
- {
- my ($k, $v) = split ("=", $_, 2);
- $Jobstep->{$k} = freezeunquote ($v) if $k;
- }
- $Jobstep->{'failures'} = 0;
- push @jobstep, $Jobstep;
-
- if ($Jobstep->{exitcode} eq "0")
- {
- push @jobstep_done, $#jobstep;
- }
- else
- {
- push @jobstep_todo, $#jobstep;
- }
- }
-
- foreach (qw (script script_version script_parameters))
- {
- $Job->{$_} = $frozenjob->{$_};
- }
- $Job->save if $job_has_uuid;
}
Log (undef, "backing off node " . $slot[$slotid]->{node}->{name} . " for 60 seconds");
}
+sub must_lock_now
+{
+ my ($lockfile, $error_message) = @_;
+ open L, ">", $lockfile or croak("$lockfile: $!");
+ if (!flock L, LOCK_EX|LOCK_NB) {
+ croak("Can't lock $lockfile: $error_message\n");
+ }
+}
+
__DATA__
#!/usr/bin/perl
--- /dev/null
+#! /usr/bin/perl
+
+use strict;
+
+use ExtUtils::MakeMaker;
+
+WriteMakefile(
+ NAME => 'Arvados',
+ VERSION_FROM => 'lib/Arvados.pm'
+);
from stream import *
import config
import errors
+import util
def normalize_stream(s, stream):
stream_tokens = [s]
class CollectionReader(object):
def __init__(self, manifest_locator_or_text):
- if re.search(r'^[a-f0-9]{32}\+\d+(\+\S)*$', manifest_locator_or_text):
+ if re.search(r'^[a-f0-9]{32}(\+\d+)?(\+\S+)*$', manifest_locator_or_text):
self._manifest_locator = manifest_locator_or_text
self._manifest_text = None
- else:
+ elif re.search(r'^\S+( [a-f0-9]{32,}(\+\S+)*)*( \d+:\d+:\S+)+\n', manifest_locator_or_text):
self._manifest_text = manifest_locator_or_text
self._manifest_locator = None
+ else:
+ raise errors.ArgumentError(
+ "Argument to CollectionReader must be a manifest or a collection UUID")
self._streams = None
def __enter__(self):
# errors.py - Arvados-specific exceptions.
+class ArgumentError(Exception):
+ pass
class SyntaxError(Exception):
pass
class AssertionError(Exception):
# wait until the driver is finished initializing
operations.initlock.wait()
+ rc = 255
try:
rc = subprocess.call(args.exec_args, shell=False)
- except:
- rc = 255
+ except OSError as e:
+ sys.stderr.write('arv-mount: %s -- exec %s\n' % (str(e), args.exec_args))
+ rc = e.errno
+ except Exception as e:
+ sys.stderr.write('arv-mount: %s\n' % str(e))
finally:
subprocess.call(["fusermount", "-u", "-z", args.mountpoint])
parser = argparse.ArgumentParser(
description='Read manifest on standard input and put normalized manifest on standard output.')
+parser.add_argument('--extract', type=str, help="The file to extract from the input manifest")
+
args = parser.parse_args()
import arvados
cr = arvados.CollectionReader(r)
-print cr.manifest_text()
+if args.extract:
+ i = args.extract.rfind('/')
+ if i == -1:
+ stream = '.'
+ fn = args.extract
+ else:
+ stream = args.extract[:i]
+ fn = args.extract[(i+1):]
+ for s in cr.all_streams():
+ if s.name() == stream:
+ if fn in s.files():
+ sys.stdout.write(s.files()[fn].as_manifest())
+else:
+ sys.stdout.write(cr.manifest_text())
os.environ['KEEP_LOCAL_STORE'] = '/tmp'
LocalCollectionWriterTest().runTest()
def runTest(self):
- cr = arvados.CollectionReader('d6c3b8e571f1b81ebb150a45ed06c884+114')
+ cr = arvados.CollectionReader('d6c3b8e571f1b81ebb150a45ed06c884+114+Xzizzle')
got = []
for s in cr.all_streams():
for f in s.all_files():
config['ARVADOS_API_HOST_INSECURE'] = ENV['ARVADOS_API_HOST_INSECURE']
config['ARVADOS_API_VERSION'] = ENV['ARVADOS_API_VERSION']
- expanded_path = File.expand_path config_file_path
- if File.exist? expanded_path
- # Load settings from the config file.
- lineno = 0
- File.open(expanded_path).each do |line|
- lineno = lineno + 1
- # skip comments and blank lines
- next if line.match('^\s*#') or not line.match('\S')
- var, val = line.chomp.split('=', 2)
- # allow environment settings to override config files.
- if var and val
- config[var] ||= val
- else
- warn "#{expanded_path}: #{lineno}: could not parse `#{line}'"
+ begin
+ expanded_path = File.expand_path config_file_path
+ if File.exist? expanded_path
+ # Load settings from the config file.
+ lineno = 0
+ File.open(expanded_path).each do |line|
+ lineno = lineno + 1
+ # skip comments and blank lines
+ next if line.match('^\s*#') or not line.match('\S')
+ var, val = line.chomp.split('=', 2)
+ # allow environment settings to override config files.
+ if var and val
+ config[var] ||= val
+ else
+ warn "#{expanded_path}: #{lineno}: could not parse `#{line}'"
+ end
end
end
+ rescue
+ debuglog "HOME environment variable (#{ENV['HOME']}) not set, not using #{config_file_path}", 0
end
@@config = config
/config/api.clinicalfuture.com.*
/config/database.yml
/config/initializers/omniauth.rb
+/config/application.yml
# asset cache
/public/assets/
# Bundle edge Rails instead:
# gem 'rails', :git => 'git://github.com/rails/rails.git'
-#gem 'sqlite3'
+group :test, :development do
+ gem 'sqlite3'
+end
+
+# This might not be needed in :test and :development, but we load it
+# anyway to make sure it always gets in Gemfile.lock and to help
+# reveal install problems sooner rather than later.
gem 'pg'
# Start using multi_json once we are on Rails 3.2;
-# Rails 3.1 has a dependency on multi_json < 1.3.0 but we need version 1.3.4 to
+# Rails 3.1 has a dependency on multi_json < 1.3.0 but we need version 1.3.4 to
# fix bug https://github.com/collectiveidea/json_spec/issues/27
gem 'multi_json'
gem 'oj'
gem 'redis'
gem 'test_after_commit', :group => :test
+
+gem 'google-api-client', '~> 0.6.3'
+gem 'trollop'
+
+gem 'themes_for_rails'
+
+gem 'arvados-cli', '>= 0.1.20140328152103'
GEM
remote: https://rubygems.org/
specs:
- actionmailer (3.2.15)
- actionpack (= 3.2.15)
+ actionmailer (3.2.17)
+ actionpack (= 3.2.17)
mail (~> 2.5.4)
- actionpack (3.2.15)
- activemodel (= 3.2.15)
- activesupport (= 3.2.15)
+ actionpack (3.2.17)
+ activemodel (= 3.2.17)
+ activesupport (= 3.2.17)
builder (~> 3.0.0)
erubis (~> 2.7.0)
journey (~> 1.0.4)
rack-cache (~> 1.2)
rack-test (~> 0.6.1)
sprockets (~> 2.2.1)
- activemodel (3.2.15)
- activesupport (= 3.2.15)
+ activemodel (3.2.17)
+ activesupport (= 3.2.17)
builder (~> 3.0.0)
- activerecord (3.2.15)
- activemodel (= 3.2.15)
- activesupport (= 3.2.15)
+ activerecord (3.2.17)
+ activemodel (= 3.2.17)
+ activesupport (= 3.2.17)
arel (~> 3.0.2)
tzinfo (~> 0.3.29)
- activeresource (3.2.15)
- activemodel (= 3.2.15)
- activesupport (= 3.2.15)
- activesupport (3.2.15)
+ activeresource (3.2.17)
+ activemodel (= 3.2.17)
+ activesupport (= 3.2.17)
+ activesupport (3.2.17)
i18n (~> 0.6, >= 0.6.4)
multi_json (~> 1.0)
- acts_as_api (0.4.1)
+ acts_as_api (0.4.2)
activemodel (>= 3.0.0)
activesupport (>= 3.0.0)
rack (>= 1.1.0)
+ addressable (2.3.6)
andand (1.3.3)
- arel (3.0.2)
+ arel (3.0.3)
+ arvados (0.1.20140414145041)
+ activesupport (>= 3.2.13)
+ andand
+ google-api-client (~> 0.6.3)
+ json (>= 1.7.7)
+ arvados-cli (0.1.20140414145041)
+ activesupport (~> 3.2, >= 3.2.13)
+ andand (~> 1.3, >= 1.3.3)
+ arvados (~> 0.1.0)
+ curb (~> 0.8)
+ google-api-client (~> 0.6.3)
+ json (~> 1.7, >= 1.7.7)
+ oj (~> 2.0, >= 2.0.3)
+ trollop (~> 2.0)
+ autoparse (0.3.3)
+ addressable (>= 2.3.1)
+ extlib (>= 0.9.15)
+ multi_json (>= 1.0.0)
builder (3.0.4)
capistrano (2.15.5)
highline
coffee-script (2.2.0)
coffee-script-source
execjs
- coffee-script-source (1.6.3)
- daemon_controller (1.1.7)
+ coffee-script-source (1.7.0)
+ curb (0.8.5)
+ daemon_controller (1.2.0)
erubis (2.7.0)
execjs (2.0.2)
- faraday (0.8.8)
+ extlib (0.9.16)
+ faraday (0.8.9)
multipart-post (~> 1.2.0)
+ google-api-client (0.6.4)
+ addressable (>= 2.3.2)
+ autoparse (>= 0.3.3)
+ extlib (>= 0.9.15)
+ faraday (~> 0.8.4)
+ jwt (>= 0.1.5)
+ launchy (>= 2.1.1)
+ multi_json (>= 1.0.0)
+ signet (~> 0.4.5)
+ uuidtools (>= 2.1.0)
hashie (1.2.0)
- highline (1.6.20)
+ highline (1.6.21)
hike (1.2.3)
- httpauth (0.2.0)
- i18n (0.6.5)
+ httpauth (0.2.1)
+ i18n (0.6.9)
journey (1.0.4)
- jquery-rails (3.0.4)
+ jquery-rails (3.1.0)
railties (>= 3.0, < 5.0)
thor (>= 0.14, < 2.0)
json (1.8.1)
- jwt (0.1.8)
+ jwt (0.1.11)
multi_json (>= 1.5)
+ launchy (2.4.2)
+ addressable (~> 2.3)
libv8 (3.16.14.3)
mail (2.5.4)
mime-types (~> 1.16)
treetop (~> 1.4.8)
- mime-types (1.25)
- multi_json (1.8.2)
+ mime-types (1.25.1)
+ multi_json (1.9.2)
multipart-post (1.2.0)
- net-scp (1.1.2)
+ net-scp (1.2.0)
net-ssh (>= 2.6.5)
net-sftp (2.1.2)
net-ssh (>= 2.6.5)
- net-ssh (2.7.0)
+ net-ssh (2.8.0)
net-ssh-gateway (1.2.0)
net-ssh (>= 2.6.5)
oauth2 (0.8.1)
jwt (~> 0.1.4)
multi_json (~> 1.0)
rack (~> 1.2)
- oj (2.1.7)
+ oj (2.7.3)
omniauth (1.1.1)
hashie (~> 1.2)
rack
omniauth-oauth2 (1.1.1)
oauth2 (~> 0.8.0)
omniauth (~> 1.0)
- passenger (4.0.23)
- daemon_controller (>= 1.1.0)
+ passenger (4.0.41)
+ daemon_controller (>= 1.2.0)
rack
rake (>= 0.8.1)
- pg (0.17.0)
- polyglot (0.3.3)
+ pg (0.17.1)
+ polyglot (0.3.4)
rack (1.4.5)
rack-cache (1.2)
rack (>= 0.4)
- rack-ssl (1.3.3)
+ rack-ssl (1.3.4)
rack
rack-test (0.6.2)
rack (>= 1.0)
- rails (3.2.15)
- actionmailer (= 3.2.15)
- actionpack (= 3.2.15)
- activerecord (= 3.2.15)
- activeresource (= 3.2.15)
- activesupport (= 3.2.15)
+ rails (3.2.17)
+ actionmailer (= 3.2.17)
+ actionpack (= 3.2.17)
+ activerecord (= 3.2.17)
+ activeresource (= 3.2.17)
+ activesupport (= 3.2.17)
bundler (~> 1.0)
- railties (= 3.2.15)
- railties (3.2.15)
- actionpack (= 3.2.15)
- activesupport (= 3.2.15)
+ railties (= 3.2.17)
+ railties (3.2.17)
+ actionpack (= 3.2.17)
+ activesupport (= 3.2.17)
rack-ssl (~> 1.3.2)
rake (>= 0.8.7)
rdoc (~> 3.4)
thor (>= 0.14.6, < 2.0)
- rake (10.1.0)
+ rake (10.2.2)
rdoc (3.12.2)
json (~> 1.4)
- redis (3.0.5)
+ redis (3.0.7)
ref (1.0.5)
rvm-capistrano (1.5.1)
capistrano (~> 2.15.4)
- sass (3.2.12)
+ sass (3.3.4)
sass-rails (3.2.6)
railties (~> 3.2.0)
sass (>= 3.1.10)
tilt (~> 1.3)
+ signet (0.4.5)
+ addressable (>= 2.2.3)
+ faraday (~> 0.8.1)
+ jwt (>= 0.1.5)
+ multi_json (>= 1.0.0)
sprockets (2.2.2)
hike (~> 1.2)
multi_json (~> 1.0)
rack (~> 1.0)
tilt (~> 1.1, != 1.3.0)
- test_after_commit (0.2.2)
- therubyracer (0.12.0)
+ sqlite3 (1.3.9)
+ test_after_commit (0.2.3)
+ themes_for_rails (0.5.1)
+ rails (>= 3.0.0)
+ therubyracer (0.12.1)
libv8 (~> 3.16.14.0)
ref
- thor (0.18.1)
+ thor (0.19.1)
tilt (1.4.1)
treetop (1.4.15)
polyglot
polyglot (>= 0.3.1)
- tzinfo (0.3.38)
- uglifier (2.3.0)
+ trollop (2.0)
+ tzinfo (0.3.39)
+ uglifier (2.5.0)
execjs (>= 0.3.0)
json (>= 1.8.0)
+ uuidtools (2.1.4)
PLATFORMS
ruby
DEPENDENCIES
acts_as_api
andand
+ arvados-cli (>= 0.1.20140328152103)
coffee-rails (~> 3.2.0)
+ google-api-client (~> 0.6.3)
jquery-rails
multi_json
oj
redis
rvm-capistrano
sass-rails (>= 3.2.0)
+ sqlite3
test_after_commit
+ themes_for_rails
therubyracer
+ trollop
uglifier (>= 1.0.3)
class ApplicationController < ActionController::Base
include CurrentApiClient
+ include ThemesForRails::ActionController
respond_to :json
protect_from_forgery
before_filter :catch_redirect_hint
before_filter :load_where_param, :only => :index
+ before_filter :load_filters_param, :only => :index
before_filter :find_objects_for_index, :only => :index
before_filter :find_object_by_uuid, :except => [:index, :create,
:render_error,
:render_error,
:render_not_found]
+ theme :select_theme
+
attr_accessor :resource_attrs
def index
def create
@object = model_class.new resource_attrs
- if @object.save
- show
- else
- render_error "Save failed"
- end
+ @object.save!
+ show
end
def update
attrs_to_update = resource_attrs.reject { |k,v|
[:kind, :etag, :href].index k
}
- if @object.update_attributes attrs_to_update
- show
- else
- render_error "Update failed"
- end
+ @object.update_attributes! attrs_to_update
+ show
end
def destroy
def render_error(e)
logger.error e.inspect
- logger.error e.backtrace.collect { |x| x + "\n" }.join('') if e.backtrace
+ if e.respond_to? :backtrace and e.backtrace
+ logger.error e.backtrace.collect { |x| x + "\n" }.join('')
+ end
if @object and @object.errors and @object.errors.full_messages and not @object.errors.full_messages.empty?
errors = @object.errors.full_messages
else
@where = params[:where]
elsif params[:where].is_a? String
begin
- @where = Oj.load(params[:where], symbol_keys: true)
+ @where = Oj.load(params[:where])
+ raise unless @where.is_a? Hash
rescue
raise ArgumentError.new("Could not parse \"where\" param as an object")
end
end
+ @where = @where.with_indifferent_access
+ end
+
+ def load_filters_param
+ @filters ||= []
+ if params[:filters].is_a? Array
+ @filters += params[:filters]
+ elsif params[:filters].is_a? String and !params[:filters].empty?
+ begin
+ f = Oj.load params[:filters]
+ raise unless f.is_a? Array
+ @filters += f
+ rescue
+ raise ArgumentError.new("Could not parse \"filters\" param as an array")
+ end
+ end
end
def find_objects_for_index
@objects ||= model_class.readable_by(current_user)
- if !@where.empty?
+ apply_where_limit_order_params
+ end
+
+ def apply_where_limit_order_params
+ if @filters.is_a? Array and @filters.any?
+ cond_out = []
+ param_out = []
+ @filters.each do |attr, operator, operand|
+ if !model_class.searchable_columns(operator).index attr.to_s
+ raise ArgumentError.new("Invalid attribute '#{attr}' in condition")
+ end
+ case operator.downcase
+ when '=', '<', '<=', '>', '>=', 'like'
+ if operand.is_a? String
+ cond_out << "#{table_name}.#{attr} #{operator} ?"
+ if (# any operator that operates on value rather than
+ # representation:
+ operator.match(/[<=>]/) and
+ model_class.attribute_column(attr).type == :datetime)
+ operand = Time.parse operand
+ end
+ param_out << operand
+ end
+ when 'in'
+ if operand.is_a? Array
+ cond_out << "#{table_name}.#{attr} IN (?)"
+ param_out << operand
+ end
+ when 'is_a'
+ operand = [operand] unless operand.is_a? Array
+ cond = []
+ operand.each do |op|
+ cl = ArvadosModel::kind_class op
+ if cl
+ cond << "#{table_name}.#{attr} like ?"
+ param_out << cl.uuid_like_pattern
+ else
+ cond << "1=0"
+ end
+ end
+ cond_out << cond.join(' OR ')
+ end
+ end
+ if cond_out.any?
+ @objects = @objects.where(cond_out.join(' AND '), *param_out)
+ end
+ end
+ if @where.is_a? Hash and @where.any?
conditions = ['1=1']
@where.each do |attr,value|
- if attr == :any
+ if attr.to_s == 'any'
if value.is_a?(Array) and
value.length == 2 and
- value[0] == 'contains' and
- model_class.columns.collect(&:name).index('name') then
+ value[0] == 'contains' then
ilikes = []
- model_class.searchable_columns.each do |column|
+ model_class.searchable_columns('ilike').each do |column|
ilikes << "#{table_name}.#{column} ilike ?"
conditions << "%#{value[1]}%"
end
where(*conditions)
end
end
+
if params[:limit]
begin
- @objects = @objects.limit(params[:limit].to_i)
+ @limit = params[:limit].to_i
rescue
raise ArgumentError.new("Invalid value for limit parameter")
end
else
- @objects = @objects.limit(100)
+ @limit = 100
end
+ @objects = @objects.limit(@limit)
+
+ orders = []
+
+ if params[:offset]
+ begin
+ @objects = @objects.offset(params[:offset].to_i)
+ @offset = params[:offset].to_i
+ rescue
+ raise ArgumentError.new("Invalid value for limit parameter")
+ end
+ else
+ @offset = 0
+ end
+
orders = []
if params[:order]
params[:order].split(',').each do |order|
if supplied_token
api_client_auth = ApiClientAuthorization.
includes(:api_client, :user).
- where('api_token=? and (expires_at is null or expires_at > now())', supplied_token).
+ where('api_token=? and (expires_at is null or expires_at > CURRENT_TIMESTAMP)', supplied_token).
first
if api_client_auth.andand.user
session[:user_id] = api_client_auth.user.id
session[:api_client_authorization_id] = api_client_auth.id
user = api_client_auth.user
api_client = api_client_auth.api_client
+ else
+ # Token seems valid, but points to a non-existent (deleted?) user.
+ api_client_auth = nil
end
elsif session[:user_id]
user = User.find(session[:user_id]) rescue nil
:kind => "arvados##{(@response_resource_name || resource_name).camelize(:lower)}List",
:etag => "",
:self_link => "",
- :next_page_token => "",
- :next_link => "",
+ :offset => @offset,
+ :limit => @limit,
:items => @objects.as_api_response(nil)
}
if @objects.respond_to? :except
- @object_list[:items_available] = @objects.except(:limit).count
+ @object_list[:items_available] = @objects.
+ except(:limit).except(:offset).
+ count(:id, distinct: true)
end
render json: @object_list
end
def self._index_requires_parameters
{
+ filters: { type: 'array', required: false },
where: { type: 'object', required: false },
order: { type: 'string', required: false }
}
end
-
+
def client_accepts_plain_text_stream
(request.headers['Accept'].split(' ') &
['text/plain', '*/*']).count > 0
end
def render *opts
- response = opts.first[:json]
- if response.is_a?(Hash) &&
- params[:_profile] &&
- Thread.current[:request_starttime]
- response[:_profile] = {
- request_time: Time.now - Thread.current[:request_starttime]
- }
+ if opts.first
+ response = opts.first[:json]
+ if response.is_a?(Hash) &&
+ params[:_profile] &&
+ Thread.current[:request_starttime]
+ response[:_profile] = {
+ request_time: Time.now - Thread.current[:request_starttime]
+ }
+ end
end
super *opts
end
+
+ def select_theme
+ return Rails.configuration.arvados_theme
+ end
end
resource_attrs[:user_id] =
User.where(uuid: resource_attrs.delete(:owner_uuid)).first.andand.id
end
+ resource_attrs[:api_client_id] = Thread.current[:api_client].id
super
end
# exist) giving the current user (or specified owner_uuid)
# permission to read it.
owner_uuid = resource_attrs.delete(:owner_uuid) || current_user.uuid
- owner_kind = if owner_uuid.match(/-(\w+)-/)[1] == User.uuid_prefix
- 'arvados#user'
- else
- 'arvados#group'
- end
unless current_user.can? write: owner_uuid
logger.warn "User #{current_user.andand.uuid} tried to set collection owner_uuid to #{owner_uuid}"
raise ArvadosModel::PermissionDeniedError
owner_uuid: owner_uuid,
link_class: 'permission',
name: 'can_read',
- head_kind: 'arvados#collection',
head_uuid: @object.uuid,
- tail_kind: owner_kind,
tail_uuid: owner_uuid
}
ActiveRecord::Base.transaction do
skip_before_filter :find_object_by_uuid, :only => :queue
skip_before_filter :render_404_if_no_object, :only => :queue
- def index
- want_ancestor = @where[:script_version_descends_from]
- if want_ancestor
- # Check for missing commit_ancestor rows, and create them if
- # possible.
- @objects.
- dup.
- includes(:commit_ancestors). # I wish Rails would let me
- # specify here which
- # commit_ancestors I am
- # interested in.
- each do |o|
- if o.commit_ancestors.
- select { |ca| ca.ancestor == want_ancestor }.
- empty? and !o.script_version.nil?
- begin
- o.commit_ancestors << CommitAncestor.find_or_create_by_descendant_and_ancestor(o.script_version, want_ancestor)
- rescue
+ def create
+ [:repository, :script, :script_version, :script_parameters].each do |r|
+ if !resource_attrs[r]
+ return render json: {
+ :error => "#{r} attribute must be specified"
+ }, status: :unprocessable_entity
+ end
+ end
+
+ r = Commit.find_commit_range(current_user,
+ resource_attrs[:repository],
+ resource_attrs[:minimum_script_version],
+ resource_attrs[:script_version],
+ resource_attrs[:exclude_script_versions])
+ if !resource_attrs[:nondeterministic] and !resource_attrs[:no_reuse]
+ # Search for jobs where the script_version is in the list of commits
+ # returned by find_commit_range
+ @object = nil
+ Job.readable_by(current_user).where(script: resource_attrs[:script],
+ script_version: r).
+ each do |j|
+ if j.nondeterministic != true and
+ j.success != false and
+ j.script_parameters == resource_attrs[:script_parameters]
+ # Record the first job in the list
+ if !@object
+ @object = j
+ end
+ # Ensure that all candidate jobs actually did produce the same output
+ if @object.output != j.output
+ @object = nil
+ break
end
end
- o.commit_ancestors.
- select { |ca| ca.ancestor == want_ancestor }.
- select(&:is).
- first
+ if @object
+ return show
+ end
end
- # Now it is safe to do an .includes().where() because we are no
- # longer interested in jobs that have other ancestors but not
- # want_ancestor.
- @objects = @objects.
- includes(:commit_ancestors).
- where('commit_ancestors.ancestor = ? and commit_ancestors.is = ?',
- want_ancestor, true)
end
+ if r
+ resource_attrs[:script_version] = r[0]
+ end
+
+ # Don't pass these on to activerecord
+ resource_attrs.delete(:minimum_script_version)
+ resource_attrs.delete(:exclude_script_versions)
+ resource_attrs.delete(:no_reuse)
super
end
def cancel
reload_object_before_update
- @object.update_attributes cancelled_at: Time.now
+ @object.update_attributes! cancelled_at: Time.now
show
end
@where.merge!({
started_at: nil,
is_locked_by_uuid: nil,
- cancelled_at: nil
+ cancelled_at: nil,
+ success: nil
})
params[:order] ||= 'priority desc, created_at'
find_objects_for_index
service_ssl_flag: true
}
end
+
def ping
params[:service_host] ||= request.env['REMOTE_ADDR']
- if not @object.ping params
- return render_not_found "object not found"
+ act_as_system_user do
+ if not @object.ping params
+ return render_not_found "object not found"
+ end
+ # Render the :superuser view (i.e., include the ping_secret) even
+ # if !current_user.is_admin. This is safe because @object.ping's
+ # success implies the ping_secret was already known by the client.
+ render json: @object.as_api_response(:superuser)
end
- # Render the :superuser view (i.e., include the ping_secret) even
- # if !current_user.is_admin. This is safe because @object.ping's
- # success implies the ping_secret was already known by the client.
- render json: @object.as_api_response(:superuser)
end
def find_objects_for_index
class Arvados::V1::LinksController < ApplicationController
- def index
- if params[:tail_uuid]
- params[:where] = Oj.load(params[:where]) if params[:where].is_a?(String)
- params[:where] ||= {}
- params[:where][:tail_uuid] = params[:tail_uuid]
+
+ def create
+ if resource_attrs[:head_kind] and ArvadosModel::resource_class_for_uuid(resource_attrs[:head_uuid]).kind != resource_attrs[:head_kind]
+ errors.add(attr, "'#{resource_attrs[:head_kind]}' does not match '#{head_uuid}'")
end
+
+ if resource_attrs[:tail_kind] and ArvadosModel::resource_class_for_uuid(resource_attrs[:tail_uuid]).kind != resource_attrs[:tail_kind]
+ errors.add(attr, "'#{resource_attrs[:tail_kind]}' does not match '#{tail_uuid}'")
+ end
+
+ resource_attrs.delete :head_kind
+ resource_attrs.delete :tail_kind
+ super
+ end
+
+ protected
+
+ # Overrides ApplicationController load_where_param
+ def load_where_param
super
+
+ # head_kind and tail_kind columns are now virtual,
+ # equivilent functionality is now provided by
+ # 'is_a', so fix up any old-style 'where' clauses.
+ if @where
+ @filters ||= []
+ if @where[:head_kind]
+ @filters << ['head_uuid', 'is_a', @where[:head_kind]]
+ @where.delete :head_kind
+ end
+ if @where[:tail_kind]
+ @filters << ['tail_uuid', 'is_a', @where[:tail_kind]]
+ @where.delete :tail_kind
+ end
+ end
+ end
+
+ # Overrides ApplicationController load_filters_param
+ def load_filters_param
+ super
+
+ # head_kind and tail_kind columns are now virtual,
+ # equivilent functionality is now provided by
+ # 'is_a', so fix up any old-style 'filter' clauses.
+ @filters = @filters.map do |k|
+ if k[0] == 'head_kind' and k[1] == '='
+ ['head_uuid', 'is_a', k[2]]
+ elsif k[0] == 'tail_kind' and k[1] == '='
+ ['tail_uuid', 'is_a', k[2]]
+ else
+ k
+ end
+ end
end
+
end
class Arvados::V1::LogsController < ApplicationController
+ # Overrides ApplicationController load_where_param
+ def load_where_param
+ super
+
+ # object_kind and column is now virtual,
+ # equivilent functionality is now provided by
+ # 'is_a', so fix up any old-style 'where' clauses.
+ if @where
+ @filters ||= []
+ if @where[:object_kind]
+ @filters << ['object_uuid', 'is_a', @where[:object_kind]]
+ @where.delete :object_kind
+ end
+ end
+ end
+
+ # Overrides ApplicationController load_filters_param
+ def load_filters_param
+ super
+
+ # object_kind and column is now virtual,
+ # equivilent functionality is now provided by
+ # 'is_a', so fix up any old-style 'filter' clauses.
+ @filters = @filters.map do |k|
+ if k[0] == 'object_kind' and k[1] == '='
+ ['object_uuid', 'is_a', k[2]]
+ else
+ k
+ end
+ end
+ end
+
end
def self._ping_requires_parameters
{ ping_secret: true }
end
+
def ping
- @object = Node.where(uuid: (params[:id] || params[:uuid])).first
- if !@object
- return render_not_found
- end
- @object.ping({ ip: params[:local_ipv4] || request.env['REMOTE_ADDR'],
- ping_secret: params[:ping_secret],
- ec2_instance_id: params[:instance_id] })
- if @object.info[:ping_secret] == params[:ping_secret]
- render json: @object.as_api_response(:superuser)
- else
- raise "Invalid ping_secret after ping"
+ act_as_system_user do
+ @object = Node.where(uuid: (params[:id] || params[:uuid])).first
+ if !@object
+ return render_not_found
+ end
+ @object.ping({ ip: params[:local_ipv4] || request.env['REMOTE_ADDR'],
+ ping_secret: params[:ping_secret],
+ ec2_instance_id: params[:instance_id] })
+ if @object.info[:ping_secret] == params[:ping_secret]
+ render json: @object.as_api_response(:superuser)
+ else
+ raise "Invalid ping_secret after ping"
+ end
end
end
gitolite_permissions = ''
perms = []
repo.permissions.each do |perm|
- if perm.tail_kind == 'arvados#group'
+ if ArvadosModel::resource_class_for_uuid(perm.tail_uuid) == Group
@users.each do |user_uuid, user|
user.group_permissions.each do |group_uuid, perm_mask|
if perm_mask[:write]
class Arvados::V1::SchemaController < ApplicationController
+ skip_before_filter :find_objects_for_index
skip_before_filter :find_object_by_uuid
skip_before_filter :render_404_if_no_object
skip_before_filter :require_auth_scope_all
- def show
- classes = Rails.cache.fetch 'arvados_v1_schema' do
- Rails.application.eager_load!
- classes = {}
- ActiveRecord::Base.descendants.reject(&:abstract_class?).each do |k|
- classes[k] = k.columns.collect do |col|
- if k.serialized_attributes.has_key? col.name
- { name: col.name,
- type: k.serialized_attributes[col.name].object_class.to_s }
- else
- { name: col.name,
- type: col.type }
- end
- end
- end
- classes
- end
- render json: classes
- end
-
- def discovery_rest_description
+ def index
expires_in 24.hours, public: true
discovery = Rails.cache.fetch 'arvados_v1_rest_discovery' do
Rails.application.eager_load!
generatedAt: Time.now.iso8601,
title: "Arvados API",
description: "The API to interact with Arvados.",
- documentationLink: "https://redmine.clinicalfuture.com/projects/arvados/",
+ documentationLink: "http://doc.arvados.org/api/index.html",
protocol: "rest",
baseUrl: root_url + "/arvados/v1/",
basePath: "/arvados/v1/",
minimum: 0,
location: "query",
},
+ offset: {
+ type: "integer",
+ description: "Number of #{k.to_s.underscore.pluralize} to skip before first returned record.",
+ default: 0,
+ format: "int32",
+ minimum: 0,
+ location: "query",
+ },
+ filters: {
+ type: "array",
+ description: "Conditions for filtering #{k.to_s.underscore.pluralize}.",
+ location: "query"
+ },
where: {
type: "object",
- description: "Conditions for filtering #{k.to_s.underscore.pluralize}.",
+ description: "Conditions for filtering #{k.to_s.underscore.pluralize}. (Deprecated. Use filters instead.)",
location: "query"
},
order: {
if httpMethod and
route.defaults[:controller] == 'arvados/v1/' + k.to_s.underscore.pluralize and
!d_methods[action.to_sym] and
- ctl_class.action_methods.include? action
+ ctl_class.action_methods.include? action and
+ ![:show, :index, :destroy].include?(action.to_sym)
method = {
id: "arvados.#{k.to_s.underscore.pluralize}.#{action}",
path: route.path.spec.to_s.sub('/arvados/v1/','').sub('(.:format)','').sub(/:(uu)?id/,'{uuid}'),
else
current_user_uuid = current_user.uuid
act_as_system_user do
- uuids = Link.where(owner_uuid: system_user_uuid,
- link_class: 'signature',
- name: 'require',
- tail_kind: 'arvados#user',
- tail_uuid: system_user_uuid,
- head_kind: 'arvados#collection').
+ uuids = Link.where("owner_uuid = ? and link_class = ? and name = ? and tail_uuid = ? and head_uuid like ?",
+ system_user_uuid,
+ 'signature',
+ 'require',
+ system_user_uuid,
+ Collection.uuid_like_pattern).
collect &:head_uuid
@objects = Collection.where('uuid in (?)', uuids)
end
current_user_uuid = (current_user.andand.is_admin && params[:uuid]) ||
current_user.uuid
act_as_system_user do
- @objects = Link.where(owner_uuid: system_user_uuid,
- link_class: 'signature',
- name: 'click',
- tail_kind: 'arvados#user',
- tail_uuid: current_user_uuid,
- head_kind: 'arvados#collection')
+ @objects = Link.where("owner_uuid = ? and link_class = ? and name = ? and tail_uuid = ? and head_uuid like ?",
+ system_user_uuid,
+ 'signature',
+ 'click',
+ current_user_uuid,
+ Collection.uuid_like_pattern)
end
@response_resource_name = 'link'
render_list
act_as_system_user do
@object = Link.create(link_class: 'signature',
name: 'click',
- tail_kind: 'arvados#user',
tail_uuid: current_user_uuid,
- head_kind: 'arvados#collection',
head_uuid: params[:uuid])
end
show
class Arvados::V1::UsersController < ApplicationController
skip_before_filter :find_object_by_uuid, only:
- [:activate, :event_stream, :current, :system]
+ [:activate, :event_stream, :current, :system, :setup]
skip_before_filter :render_404_if_no_object, only:
- [:activate, :event_stream, :current, :system]
+ [:activate, :event_stream, :current, :system, :setup]
+ before_filter :admin_required, only: [:setup, :unsetup]
def current
@object = current_user
end
end
end
-
+
def event_stream
channel = current_user.andand.uuid
if current_user.andand.is_admin
raise ArgumentError.new "Cannot activate without being invited."
end
act_as_system_user do
- required_uuids = Link.where(owner_uuid: system_user_uuid,
- link_class: 'signature',
- name: 'require',
- tail_uuid: system_user_uuid,
- head_kind: 'arvados#collection').
+ required_uuids = Link.where("owner_uuid = ? and link_class = ? and name = ? and tail_uuid = ? and head_uuid like ?",
+ system_user_uuid,
+ 'signature',
+ 'require',
+ system_user_uuid,
+ Collection.uuid_like_pattern).
collect(&:head_uuid)
signed_uuids = Link.where(owner_uuid: system_user_uuid,
link_class: 'signature',
name: 'click',
- tail_kind: 'arvados#user',
tail_uuid: @object.uuid,
- head_kind: 'arvados#collection',
head_uuid: required_uuids).
collect(&:head_uuid)
todo_uuids = required_uuids - signed_uuids
end
show
end
+
+ # create user object and all the needed links
+ def setup
+ @object = nil
+ if params[:uuid]
+ @object = User.find_by_uuid params[:uuid]
+ if !@object
+ return render_404_if_no_object
+ end
+ object_found = true
+ else
+ if !params[:user]
+ raise ArgumentError.new "Required uuid or user"
+ else
+ if params[:user]['uuid']
+ @object = User.find_by_uuid params[:user]['uuid']
+ if @object
+ object_found = true
+ end
+ end
+
+ if !@object
+ if !params[:user]['email']
+ raise ArgumentError.new "Require user email"
+ end
+
+ if !params[:openid_prefix]
+ raise ArgumentError.new "Required openid_prefix parameter is missing."
+ end
+
+ @object = model_class.create! resource_attrs
+ end
+ end
+ end
+
+ if object_found
+ @response = @object.setup_repo_vm_links params[:repo_name],
+ params[:vm_uuid], params[:openid_prefix]
+ else
+ @response = User.setup @object, params[:openid_prefix],
+ params[:repo_name], params[:vm_uuid]
+ end
+
+ # setup succeeded. send email to user
+ if params[:send_notification_email] == true || params[:send_notification_email] == 'true'
+ UserNotifier.account_is_setup(@object).deliver
+ end
+
+ render json: { kind: "arvados#HashList", items: @response.as_api_response(nil) }
+ end
+
+ # delete user agreements, vm, repository, login links; set state to inactive
+ def unsetup
+ reload_object_before_update
+ @object.unsetup
+ show
+ end
+
+ protected
+
+ def self._setup_requires_parameters
+ {
+ send_notification_email: { type: 'boolean', required: true },
+ }
+ end
+
end
if not user
# Check for permission to log in to an existing User record with
# a different identity_url
- Link.where(link_class: 'permission',
- name: 'can_login',
- tail_kind: 'email',
- tail_uuid: omniauth['info']['email'],
- head_kind: 'arvados#user').each do |link|
+ Link.where("link_class = ? and name = ? and tail_uuid = ? and head_uuid like ?",
+ 'permission',
+ 'can_login',
+ omniauth['info']['email'],
+ User.uuid_like_pattern).each do |link|
if prefix = link.properties['identity_url_prefix']
if prefix == omniauth['info']['identity_url'][0..prefix.size-1]
user = User.find_by_uuid(link.head_uuid)
api_client_auth = ApiClientAuthorization.
new(user: user,
api_client: @api_client,
- created_by_ip_address: remote_ip)
+ created_by_ip_address: remote_ip,
+ scopes: ["all"])
api_client_auth.save!
if callback_url.index('?')
--- /dev/null
+class UserNotifier < ActionMailer::Base
+ default from: Rails.configuration.user_notifier_email_from
+
+ def account_is_setup(user)
+ @user = user
+ mail(to: user.email, subject: 'Welcome to Curoverse')
+ end
+end
t.add :scopes
end
+ UNLOGGED_CHANGES = ['last_used_at', 'last_used_by_ip_address', 'updated_at']
+
def assign_random_api_token
self.api_token ||= rand(2**256).to_s(36)
end
end
def modified_at=(x) end
+ def logged_attributes
+ attrs = attributes.dup
+ attrs.delete('api_token')
+ attrs
+ end
+
protected
def permission_to_create
not self.user_id_changed? and
not self.owner_uuid_changed?)
end
+
+ def log_update
+ super unless (changed - UNLOGGED_CHANGES).empty?
+ end
end
attr_protected :modified_by_user_uuid
attr_protected :modified_by_client_uuid
attr_protected :modified_at
+ after_initialize :log_start_state
before_create :ensure_permission_to_create
before_update :ensure_permission_to_update
before_destroy :ensure_permission_to_destroy
+
before_create :update_modified_by_fields
before_update :maybe_update_modified_by_fields
+ after_create :log_create
+ after_update :log_update
+ after_destroy :log_destroy
validate :ensure_serialized_attribute_type
validate :normalize_collection_uuids
+ validate :ensure_valid_uuids
has_many :permissions, :foreign_key => :head_uuid, :class_name => 'Link', :primary_key => :uuid, :conditions => "link_class = 'permission'"
end
def self.kind_class(kind)
- kind.match(/^arvados\#(.+?)(_list|List)?$/)[1].pluralize.classify.constantize rescue nil
+ kind.match(/^arvados\#(.+)$/)[1].classify.safe_constantize rescue nil
end
def href
"#{current_api_base}/#{self.class.to_s.pluralize.underscore}/#{self.uuid}"
end
- def self.searchable_columns
+ def self.searchable_columns operator
+ textonly_operator = !operator.match(/[<=>]/)
self.columns.collect do |col|
- if [:string, :text].index(col.type) && col.name != 'owner_uuid'
+ if col.name == 'owner_uuid'
+ nil
+ elsif [:string, :text].index(col.type)
+ col.name
+ elsif !textonly_operator and [:datetime, :integer].index(col.type)
col.name
end
end.compact
end
- def eager_load_associations
- self.class.columns.each do |col|
- re = col.name.match /^(.*)_kind$/
- if (re and
- self.respond_to? re[1].to_sym and
- (auuid = self.send((re[1] + '_uuid').to_sym)) and
- (aclass = self.class.kind_class(self.send(col.name.to_sym))) and
- (aobject = aclass.where('uuid=?', auuid).first))
- self.instance_variable_set('@'+re[1], aobject)
- end
- end
+ def self.attribute_column attr
+ self.columns.select { |col| col.name == attr.to_s }.first
end
+ # def eager_load_associations
+ # self.class.columns.each do |col|
+ # re = col.name.match /^(.*)_kind$/
+ # if (re and
+ # self.respond_to? re[1].to_sym and
+ # (auuid = self.send((re[1] + '_uuid').to_sym)) and
+ # (aclass = self.class.kind_class(self.send(col.name.to_sym))) and
+ # (aobject = aclass.where('uuid=?', auuid).first))
+ # self.instance_variable_set('@'+re[1], aobject)
+ # end
+ # end
+ # end
+
def self.readable_by user
uuid_list = [user.uuid, *user.groups_i_can(:read)]
sanitized_uuid_list = uuid_list.
user.uuid)
end
+ def logged_attributes
+ attributes
+ end
+
protected
def ensure_permission_to_create
end
def maybe_update_modified_by_fields
- update_modified_by_fields if self.changed?
+ update_modified_by_fields if self.changed? or self.new_record?
end
def update_modified_by_fields
self.created_at ||= Time.now
- self.owner_uuid ||= current_default_owner
+ self.owner_uuid ||= current_default_owner if self.respond_to? :owner_uuid=
self.modified_at = Time.now
self.modified_by_user_uuid = current_user ? current_user.uuid : nil
self.modified_by_client_uuid = current_api_client ? current_api_client.uuid : nil
attributes.keys.select { |a| a.match /_uuid$/ }
end
+ def skip_uuid_read_permission_check
+ %w(modified_by_client_uuid)
+ end
+
def normalize_collection_uuids
foreign_key_attributes.each do |attr|
attr_value = send attr
end
end
+ @@UUID_REGEX = /^[0-9a-z]{5}-([0-9a-z]{5})-[0-9a-z]{15}$/
+
+ @@prefixes_hash = nil
+ def self.uuid_prefixes
+ unless @@prefixes_hash
+ @@prefixes_hash = {}
+ ActiveRecord::Base.descendants.reject(&:abstract_class?).each do |k|
+ if k.respond_to?(:uuid_prefix)
+ @@prefixes_hash[k.uuid_prefix] = k
+ end
+ end
+ end
+ @@prefixes_hash
+ end
+
+ def self.uuid_like_pattern
+ "_____-#{uuid_prefix}-_______________"
+ end
+
+ def ensure_valid_uuids
+ specials = [system_user_uuid, 'd41d8cd98f00b204e9800998ecf8427e+0']
+
+ foreign_key_attributes.each do |attr|
+ if new_record? or send (attr + "_changed?")
+ attr_value = send attr
+ r = ArvadosModel::resource_class_for_uuid attr_value if attr_value
+ r = r.readable_by(current_user) if r and not skip_uuid_read_permission_check.include? attr
+ if r and r.where(uuid: attr_value).count == 0 and not specials.include? attr_value
+ errors.add(attr, "'#{attr_value}' not found")
+ end
+ end
+ end
+ end
+
+ class Email
+ def self.kind
+ "email"
+ end
+
+ def kind
+ self.class.kind
+ end
+
+ def self.readable_by (u)
+ self
+ end
+
+ def self.where (u)
+ [{:uuid => u[:uuid]}]
+ end
+ end
+
def self.resource_class_for_uuid(uuid)
if uuid.is_a? ArvadosModel
return uuid.class
resource_class = nil
Rails.application.eager_load!
- uuid.match /^[0-9a-z]{5}-([0-9a-z]{5})-[0-9a-z]{15}$/ do |re|
- ActiveRecord::Base.descendants.reject(&:abstract_class?).each do |k|
- if k.respond_to?(:uuid_prefix)
- if k.uuid_prefix == re[1]
- return k
- end
- end
- end
+ uuid.match @@UUID_REGEX do |re|
+ return uuid_prefixes[re[1]] if uuid_prefixes[re[1]]
+ end
+
+ if uuid.match /.+@.+/
+ return Email
end
+
nil
end
+ def log_start_state
+ @old_etag = etag
+ @old_attributes = logged_attributes
+ end
+
+ def log_change(event_type)
+ log = Log.new(event_type: event_type).fill_object(self)
+ yield log
+ log.save!
+ log_start_state
+ end
+
+ def log_create
+ log_change('create') do |log|
+ log.fill_properties('old', nil, nil)
+ log.update_to self
+ end
+ end
+
+ def log_update
+ log_change('update') do |log|
+ log.fill_properties('old', @old_etag, @old_attributes)
+ log.update_to self
+ end
+ end
+
+ def log_destroy
+ log_change('destroy') do |log|
+ log.fill_properties('old', @old_etag, @old_attributes)
+ log.update_to nil
+ end
+ end
end
--- /dev/null
+class Blob
+
+ # In order to get a Blob from Keep, you have to prove either
+ # [a] you have recently written it to Keep yourself, or
+ # [b] apiserver has recently decided that you should be able to read it
+ #
+ # To ensure that the requestor of a blob is authorized to read it,
+ # Keep requires clients to timestamp the blob locator with an expiry
+ # time, and to sign the timestamped locator with their API token.
+ #
+ # A signed blob locator has the form:
+ # locator_hash +A blob_signature @ timestamp
+ # where the timestamp is a Unix time expressed as a hexadecimal value,
+ # and the blob_signature is the signed locator_hash + API token + timestamp.
+ #
+ class InvalidSignatureError < StandardError
+ end
+
+ # Blob.sign_locator: return a signed and timestamped blob locator.
+ #
+ # The 'opts' argument should include:
+ # [required] :key - the Arvados server-side blobstore key
+ # [required] :api_token - user's API token
+ # [optional] :ttl - number of seconds before this request expires
+ #
+ def self.sign_locator blob_locator, opts
+ # We only use the hash portion for signatures.
+ blob_hash = blob_locator.split('+').first
+
+ # Generate an expiry timestamp (seconds since epoch, base 16)
+ timestamp = (Time.now.to_i + (opts[:ttl] || 600)).to_s(16)
+ # => "53163cb4"
+
+ # Generate a signature.
+ signature =
+ generate_signature opts[:key], blob_hash, opts[:api_token], timestamp
+
+ blob_locator + '+A' + signature + '@' + timestamp
+ end
+
+ # Blob.verify_signature
+ # Safely verify the signature on a blob locator.
+ # Return value: true if the locator has a valid signature, false otherwise
+ # Arguments: signed_blob_locator, opts
+ #
+ def self.verify_signature *args
+ begin
+ self.verify_signature! *args
+ true
+ rescue Blob::InvalidSignatureError
+ false
+ end
+ end
+
+ # Blob.verify_signature!
+ # Verify the signature on a blob locator.
+ # Return value: true if the locator has a valid signature
+ # Arguments: signed_blob_locator, opts
+ # Exceptions:
+ # Blob::InvalidSignatureError if the blob locator does not include a
+ # valid signature
+ #
+ def self.verify_signature! signed_blob_locator, opts
+ blob_hash = signed_blob_locator.split('+').first
+ given_signature, timestamp = signed_blob_locator.
+ split('+A').last.
+ split('+').first.
+ split('@')
+
+ if !timestamp
+ raise Blob::InvalidSignatureError.new 'No signature provided.'
+ end
+ if !timestamp.match /^[\da-f]+$/
+ raise Blob::InvalidSignatureError.new 'Timestamp is not a base16 number.'
+ end
+ if timestamp.to_i(16) < Time.now.to_i
+ raise Blob::InvalidSignatureError.new 'Signature expiry time has passed.'
+ end
+
+ my_signature =
+ generate_signature opts[:key], blob_hash, opts[:api_token], timestamp
+
+ if my_signature != given_signature
+ raise Blob::InvalidSignatureError.new 'Signature is invalid.'
+ end
+
+ true
+ end
+
+ def self.generate_signature key, blob_hash, api_token, timestamp
+ OpenSSL::HMAC.hexdigest('sha1', key,
+ [blob_hash,
+ api_token,
+ timestamp].join('@'))
+ end
+end
end
end
+ def self.uuid_like_pattern
+ "________________________________+%"
+ end
+
def self.normalize_uuid uuid
hash_part = nil
size_part = nil
class Commit < ActiveRecord::Base
require 'shellwords'
- # Make sure the specified commit really exists, and return the full
- # sha1 commit hash.
- #
- # Accepts anything "git rev-list" accepts, optionally (and
- # preferably) preceded by "repo_name:".
- #
- # Examples: "1234567", "master", "apps:1234567", "apps:master",
- # "apps:HEAD"
-
- def self.find_by_commit_ish(commit_ish)
- want_repo = nil
- if commit_ish.index(':')
- want_repo, commit_ish = commit_ish.split(':',2)
+ def self.git_check_ref_format(e)
+ if !e or e.empty? or e[0] == '-' or e[0] == '$'
+ # definitely not valid
+ false
+ else
+ `git check-ref-format --allow-onelevel #{e.shellescape}`
+ $?.success?
end
- repositories.each do |repo_name, repo|
- next if want_repo and want_repo != repo_name
- ENV['GIT_DIR'] = repo[:git_dir]
- IO.foreach("|git rev-list --max-count=1 --format=oneline 'origin/'#{commit_ish.shellescape} 2>/dev/null || git rev-list --max-count=1 --format=oneline ''#{commit_ish.shellescape}") do |line|
- sha1, message = line.strip.split " ", 2
- next if sha1.length != 40
- begin
- Commit.find_or_create_by_repository_name_and_sha1_and_message(repo_name, sha1, message[0..254])
- rescue
- logger.warn "find_or_create failed: repo_name #{repo_name} sha1 #{sha1} message #{message[0..254]}"
- # Ignore cache failure. Commit is real. We should proceed.
+ end
+
+ def self.find_commit_range(current_user, repository, minimum, maximum, exclude)
+ if (minimum and !git_check_ref_format(minimum)) or !git_check_ref_format(maximum)
+ logger.warn "find_commit_range called with invalid minimum or maximum: '#{minimum}', '#{maximum}'"
+ return nil
+ end
+
+ if minimum and minimum.empty?
+ minimum = nil
+ end
+
+ if !maximum
+ maximum = "HEAD"
+ end
+
+ # Get list of actual repository directories under management
+ on_disk_repos = repositories
+
+ # Get list of repository objects readable by user
+ readable = Repository.readable_by(current_user)
+
+ # filter repository objects on requested repository name
+ if repository
+ readable = readable.where(name: repository)
+ end
+
+ commits = []
+ readable.each do |r|
+ if on_disk_repos[r.name]
+ ENV['GIT_DIR'] = on_disk_repos[r.name][:git_dir]
+
+ # We've filtered for invalid characters, so we can pass the contents of
+ # minimum and maximum safely on the command line
+
+ # Get the commit hash for the upper bound
+ max_hash = nil
+ IO.foreach("|git rev-list --max-count=1 #{maximum.shellescape}") do |line|
+ max_hash = line.strip
+ end
+
+ # If not found or string is invalid, nothing else to do
+ next if !max_hash or !git_check_ref_format(max_hash)
+
+ resolved_exclude = nil
+ if exclude
+ resolved_exclude = []
+ exclude.each do |e|
+ if git_check_ref_format(e)
+ IO.foreach("|git rev-list --max-count=1 #{e.shellescape}") do |line|
+ resolved_exclude.push(line.strip)
+ end
+ else
+ logger.warn "find_commit_range called with invalid exclude invalid characters: '#{exclude}'"
+ return nil
+ end
+ end
+ end
+
+ if minimum
+ # Get the commit hash for the lower bound
+ min_hash = nil
+ IO.foreach("|git rev-list --max-count=1 #{minimum.shellescape}") do |line|
+ min_hash = line.strip
+ end
+
+ # If not found or string is invalid, nothing else to do
+ next if !min_hash or !git_check_ref_format(min_hash)
+
+ # Now find all commits between them
+ IO.foreach("|git rev-list #{min_hash.shellescape}..#{max_hash.shellescape}") do |line|
+ hash = line.strip
+ commits.push(hash) if !resolved_exclude or !resolved_exclude.include? hash
+ end
+
+ commits.push(min_hash) if !resolved_exclude or !resolved_exclude.include? min_hash
+ else
+ commits.push(max_hash) if !resolved_exclude or !resolved_exclude.include? max_hash
end
- return sha1
end
end
- nil
+
+ if !commits or commits.empty?
+ nil
+ else
+ commits
+ end
end
# Import all commits from configured git directory into the commits
end
end
+ def self.refresh_repositories
+ @repositories = nil
+ end
+
protected
def self.repositories
next if repo.match /^\./
git_dir = File.join(@gitdirbase,
repo.match(/\.git$/) ? repo : File.join(repo, '.git'))
+ next if git_dir == Rails.configuration.git_internal_dir
repo_name = repo.sub(/\.git$/, '')
@repositories[repo_name] = {git_dir: git_dir}
end
t.add :started_at
t.add :finished_at
t.add :output
+ t.add :output_is_persistent
t.add :success
t.add :running
t.add :is_locked_by_uuid
t.add :dependencies
t.add :log_stream_href
t.add :log_buffer
+ t.add :nondeterministic
+ t.add :repository
end
def assert_finished
end
def self.queue
- self.where('started_at is ? and is_locked_by_uuid is ? and cancelled_at is ?',
- nil, nil, nil).
+ self.where('started_at is ? and is_locked_by_uuid is ? and cancelled_at is ? and success is ?',
+ nil, nil, nil, nil).
+ order('priority desc, created_at')
+ end
+
+ def self.running
+ self.where('running = ?', true).
order('priority desc, created_at')
end
super + %w(output log)
end
+ def skip_uuid_read_permission_check
+ super + %w(cancelled_by_client_uuid)
+ end
+
def ensure_script_version_is_commit
if self.is_locked_by_uuid and self.started_at
# Apparently client has already decided to go for it. This is
return true
end
if new_record? or script_version_changed?
- sha1 = Commit.find_by_commit_ish(self.script_version) rescue nil
+ sha1 = Commit.find_commit_range(current_user, nil, nil, self.script_version, nil)[0] rescue nil
if sha1
self.script_version = sha1
else
def permission_to_update
if is_locked_by_uuid_was and !(current_user and
- current_user.uuid == is_locked_by_uuid_was)
+ (current_user.uuid == is_locked_by_uuid_was or
+ current_user.uuid == system_user.uuid))
if script_changed? or
script_parameters_changed? or
script_version_changed? or
t.add :ping_secret
end
+ def foreign_key_attributes
+ super.reject { |a| a == "filesystem_uuid" }
+ end
+
def ping(o)
raise "must have :service_host and :ping_secret" unless o[:service_host] and o[:ping_secret]
end
@bypass_arvados_authorization = true
- self.update_attributes(o.select { |k,v|
+ self.update_attributes!(o.select { |k,v|
[:service_host,
:service_port,
:service_ssl_flag,
after_update :maybe_invalidate_permissions_cache
after_create :maybe_invalidate_permissions_cache
after_destroy :maybe_invalidate_permissions_cache
-
- attr_accessor :head
- attr_accessor :tail
+ attr_accessor :head_kind, :tail_kind
api_accessible :user, extend: :common do |t|
- t.add :tail_kind
t.add :tail_uuid
t.add :link_class
t.add :name
- t.add :head_kind
t.add :head_uuid
- t.add :head, :if => :head
- t.add :tail, :if => :tail
+ t.add :head_kind
+ t.add :tail_kind
t.add :properties
end
super
end
+ def head_kind
+ if k = ArvadosModel::resource_class_for_uuid(head_uuid)
+ k.kind
+ end
+ end
+
+ def tail_kind
+ if k = ArvadosModel::resource_class_for_uuid(tail_uuid)
+ k.kind
+ end
+ end
+
protected
def permission_to_attach_to_objects
# All users can grant permissions on objects they own
head_obj = self.class.
- kind_class(self.head_kind).
+ kind_class(self.head_uuid).
where('uuid=?',head_uuid).
first
if head_obj
include AssignUuid
include KindAndEtag
include CommonApiTemplate
- serialize :info, Hash
+ serialize :properties, Hash
before_validation :set_default_event_at
- attr_accessor :object
+ attr_accessor :object, :object_kind
api_accessible :user, extend: :common do |t|
- t.add :object_kind
t.add :object_uuid
t.add :object, :if => :object
+ t.add :object_kind
t.add :event_at
t.add :event_type
t.add :summary
- t.add :info
+ t.add :properties
+ end
+
+ def object_kind
+ if k = ArvadosModel::resource_class_for_uuid(object_uuid)
+ k.kind
+ end
+ end
+
+ def fill_object(thing)
+ self.object_uuid ||= thing.uuid
+ self.summary ||= "#{self.event_type} of #{thing.uuid}"
+ self
+ end
+
+ def fill_properties(age, etag_prop, attrs_prop)
+ self.properties.merge!({"#{age}_etag" => etag_prop,
+ "#{age}_attributes" => attrs_prop})
+ end
+
+ def update_to(thing)
+ fill_properties('new', thing.andand.etag, thing.andand.logged_attributes)
+ case event_type
+ when "create"
+ self.event_at = thing.created_at
+ when "update"
+ self.event_at = thing.modified_at
+ when "destroy"
+ self.event_at = Time.now
+ end
+ self
end
protected
+ def permission_to_create
+ true
+ end
+
+ def permission_to_update
+ current_user.andand.is_admin
+ end
+
+ alias_method :permission_to_delete, :permission_to_update
+
def set_default_event_at
self.event_at ||= Time.now
end
+
+ def log_change(event_type)
+ # Don't log changes to logs.
+ end
+
+ def ensure_valid_uuids
+ # logs can have references to deleted objects
+ end
end
MAX_SLOTS = 64
- @@confdir = if Rails.configuration.respond_to? :dnsmasq_conf_dir
- Rails.configuration.dnsmasq_conf_dir
- elsif File.exists? '/etc/dnsmasq.d/.'
- '/etc/dnsmasq.d'
- else
- nil
- end
+ @@confdir = Rails.configuration.dnsmasq_conf_dir
@@domain = Rails.configuration.compute_node_domain rescue `hostname --domain`.strip
@@nameservers = Rails.configuration.compute_node_nameservers
def start!(ping_url_method)
ensure_permission_to_update
ping_url = ping_url_method.call({ uuid: self.uuid, ping_secret: self.info[:ping_secret] })
- if (Rails.configuration.compute_node_ec2run_args rescue false) and
- (Rails.configuration.compute_node_ami rescue false)
+ if (Rails.configuration.compute_node_ec2run_args and
+ Rails.configuration.compute_node_ami)
ec2_args = ["--user-data '#{ping_url}'",
"-t c1.xlarge -n 1",
Rails.configuration.compute_node_ec2run_args,
t.collect { |r| r[2] }.inject(0.0) { |sum,a| sum += a } / t.size
end
+ def self.queue
+ self.where('active = true')
+ end
+
protected
def bootstrap_components
if pipeline_template and (!components or components.empty?)
before_update :prevent_privilege_escalation
before_update :prevent_inactive_admin
before_create :check_auto_admin
+ after_create :add_system_group_permission_link
after_create AdminNotifier
has_many :authorized_keys, :foreign_key => :authorized_user_uuid, :primary_key => :uuid
Group.where('owner_uuid in (?)', lookup_uuids).each do |group|
newgroups << [group.owner_uuid, group.uuid, 'can_manage']
end
- Link.where('tail_uuid in (?) and link_class = ? and head_kind = ?',
+ Link.where('tail_uuid in (?) and link_class = ? and (head_uuid like ? or head_uuid like ?)',
lookup_uuids,
'permission',
- 'arvados#group').each do |link|
+ Group.uuid_like_pattern,
+ User.uuid_like_pattern).each do |link|
newgroups << [link.tail_uuid, link.head_uuid, link.name]
end
newgroups.each do |tail_uuid, head_uuid, perm_name|
end
end
+ def self.setup(user, openid_prefix, repo_name=nil, vm_uuid=nil)
+ return user.setup_repo_vm_links(repo_name, vm_uuid, openid_prefix)
+ end
+
+ # create links
+ def setup_repo_vm_links(repo_name, vm_uuid, openid_prefix)
+ oid_login_perm = create_oid_login_perm openid_prefix
+ repo_perm = create_user_repo_link repo_name
+ vm_login_perm = create_vm_login_permission_link vm_uuid, repo_name
+ group_perm = create_user_group_link
+
+ return [oid_login_perm, repo_perm, vm_login_perm, group_perm, self].compact
+ end
+
+ # delete user signatures, login, repo, and vm perms, and mark as inactive
+ def unsetup
+ # delete oid_login_perms for this user
+ oid_login_perms = Link.where(tail_uuid: self.email,
+ link_class: 'permission',
+ name: 'can_login')
+ oid_login_perms.each do |perm|
+ Link.delete perm
+ end
+
+ # delete repo_perms for this user
+ repo_perms = Link.where(tail_uuid: self.uuid,
+ link_class: 'permission',
+ name: 'can_write')
+ repo_perms.each do |perm|
+ Link.delete perm
+ end
+
+ # delete vm_login_perms for this user
+ vm_login_perms = Link.where(tail_uuid: self.uuid,
+ link_class: 'permission',
+ name: 'can_login')
+ vm_login_perms.each do |perm|
+ Link.delete perm
+ end
+
+ # delete "All users' group read permissions for this user
+ group = Group.where(name: 'All users').select do |g|
+ g[:uuid].match /-f+$/
+ end.first
+ group_perms = Link.where(tail_uuid: self.uuid,
+ head_uuid: group[:uuid],
+ link_class: 'permission',
+ name: 'can_read')
+ group_perms.each do |perm|
+ Link.delete perm
+ end
+
+ # delete any signatures by this user
+ signed_uuids = Link.where(link_class: 'signature',
+ tail_uuid: self.uuid)
+ signed_uuids.each do |sign|
+ Link.delete sign
+ end
+
+ # mark the user as inactive
+ self.is_active = false
+ self.save!
+ end
+
protected
def permission_to_update
end
def check_auto_admin
- if User.where("uuid not like '%-000000000000000'").where(:is_admin => true).count == 0 and not Rails.configuration.auto_admin_user.nil?
+ if User.where("uuid not like '%-000000000000000'").where(:is_admin => true).count == 0 and Rails.configuration.auto_admin_user
if current_user.email == Rails.configuration.auto_admin_user
self.is_admin = true
self.is_active = true
upstream_path.delete start
merged
end
+
+ def create_oid_login_perm (openid_prefix)
+ login_perm_props = {identity_url_prefix: openid_prefix}
+
+ # Check oid_login_perm
+ oid_login_perms = Link.where(tail_uuid: self.email,
+ link_class: 'permission',
+ name: 'can_login').where("head_uuid like ?", User.uuid_like_pattern)
+
+ if !oid_login_perms.any?
+ # create openid login permission
+ oid_login_perm = Link.create(link_class: 'permission',
+ name: 'can_login',
+ tail_uuid: self.email,
+ head_uuid: self.uuid,
+ properties: login_perm_props
+ )
+ logger.info { "openid login permission: " + oid_login_perm[:uuid] }
+ else
+ oid_login_perm = oid_login_perms.first
+ end
+
+ return oid_login_perm
+ end
+
+ def create_user_repo_link(repo_name)
+ # repo_name is optional
+ if not repo_name
+ logger.warn ("Repository name not given for #{self.uuid}.")
+ return
+ end
+
+ # Check for an existing repository with the same name we're about to use.
+ repo = Repository.where(name: repo_name).first
+
+ if repo
+ logger.warn "Repository exists for #{repo_name}: #{repo[:uuid]}."
+
+ # Look for existing repository access for this repo
+ repo_perms = Link.where(tail_uuid: self.uuid,
+ head_uuid: repo[:uuid],
+ link_class: 'permission',
+ name: 'can_write')
+ if repo_perms.any?
+ logger.warn "User already has repository access " +
+ repo_perms.collect { |p| p[:uuid] }.inspect
+ return repo_perms.first
+ end
+ end
+
+ # create repo, if does not already exist
+ repo ||= Repository.create(name: repo_name)
+ logger.info { "repo uuid: " + repo[:uuid] }
+
+ repo_perm = Link.create(tail_uuid: self.uuid,
+ head_uuid: repo[:uuid],
+ link_class: 'permission',
+ name: 'can_write')
+ logger.info { "repo permission: " + repo_perm[:uuid] }
+ return repo_perm
+ end
+
+ # create login permission for the given vm_uuid, if it does not already exist
+ def create_vm_login_permission_link(vm_uuid, repo_name)
+ begin
+
+ # vm uuid is optional
+ if vm_uuid
+ vm = VirtualMachine.where(uuid: vm_uuid).first
+
+ if not vm
+ logger.warn "Could not find virtual machine for #{vm_uuid.inspect}"
+ raise "No vm found for #{vm_uuid}"
+ end
+ else
+ return
+ end
+
+ logger.info { "vm uuid: " + vm[:uuid] }
+
+ login_perms = Link.where(tail_uuid: self.uuid,
+ head_uuid: vm[:uuid],
+ link_class: 'permission',
+ name: 'can_login')
+
+ perm_exists = false
+ login_perms.each do |perm|
+ if perm.properties[:username] == repo_name
+ perm_exists = true
+ break
+ end
+ end
+
+ if !perm_exists
+ login_perm = Link.create(tail_uuid: self.uuid,
+ head_uuid: vm[:uuid],
+ link_class: 'permission',
+ name: 'can_login',
+ properties: {username: repo_name})
+ logger.info { "login permission: " + login_perm[:uuid] }
+ else
+ login_perm = login_perms.first
+ end
+
+ return login_perm
+ end
+ end
+
+ # add the user to the 'All users' group
+ def create_user_group_link
+ # Look up the "All users" group (we expect uuid *-*-fffffffffffffff).
+ group = Group.where(name: 'All users').select do |g|
+ g[:uuid].match /-f+$/
+ end.first
+
+ if not group
+ logger.warn "No 'All users' group with uuid '*-*-fffffffffffffff'."
+ raise "No 'All users' group with uuid '*-*-fffffffffffffff' is found"
+ else
+ logger.info { "\"All users\" group uuid: " + group[:uuid] }
+
+ group_perms = Link.where(tail_uuid: self.uuid,
+ head_uuid: group[:uuid],
+ link_class: 'permission',
+ name: 'can_read')
+
+ if !group_perms.any?
+ group_perm = Link.create(tail_uuid: self.uuid,
+ head_uuid: group[:uuid],
+ link_class: 'permission',
+ name: 'can_read')
+ logger.info { "group permission: " + group_perm[:uuid] }
+ else
+ group_perm = group_perms.first
+ end
+
+ return group_perm
+ end
+ end
+
+ # Give the special "System group" permission to manage this user and
+ # all of this user's stuff.
+ #
+ def add_system_group_permission_link
+ act_as_system_user do
+ Link.create(link_class: 'permission',
+ name: 'can_manage',
+ tail_uuid: system_group_uuid,
+ head_uuid: self.uuid)
+ end
+ end
end
--- /dev/null
+<%= @user.full_name %>,
+
+Your Arvados account has been set up. You can log in here using your
+Google account (<%= @user.email %>):
+
+<%= Rails.configuration.workbench_address %>
--- /dev/null
+# Do not use this file for site configuration. Create application.yml
+# instead (see application.yml.example).
+
+development:
+ force_ssl: false
+ cache_classes: false
+ whiny_nils: true
+ consider_all_requests_local: true
+ action_controller.perform_caching: false
+ action_mailer.raise_delivery_errors: false
+ action_mailer.perform_deliveries: false
+ active_support.deprecation: :log
+ action_dispatch.best_standards_support: :builtin
+ active_record.mass_assignment_sanitizer: :strict
+ active_record.auto_explain_threshold_in_seconds: 0.5
+ assets.compress: false
+ assets.debug: true
+
+production:
+ force_ssl: true
+ cache_classes: true
+ consider_all_requests_local: false
+ action_controller.perform_caching: true
+ serve_static_assets: false
+ assets.compress: true
+ assets.compile: false
+ assets.digest: true
+
+test:
+ force_ssl: false
+ cache_classes: true
+ serve_static_assets: true
+ static_cache_control: public, max-age=3600
+ whiny_nils: true
+ consider_all_requests_local: true
+ action_controller.perform_caching: false
+ action_dispatch.show_exceptions: false
+ action_controller.allow_forgery_protection: false
+ action_mailer.delivery_method: :test
+ active_support.deprecation: :stderr
+ active_record.mass_assignment_sanitizer: :strict
+ uuid_prefix: zzzzz
+
+common:
+ secret_token: ~
+ uuid_prefix: <%= Digest::MD5.hexdigest(`hostname`).to_i(16).to_s(36)[0..4] %>
+
+ # Git repositories must be readable by api server, or you won't be
+ # able to submit crunch jobs. To pass the test suites, put a clone
+ # of the arvados tree in {git_repositories_dir}/arvados.git or
+ # {git_repositories_dir}/arvados/.git
+ git_repositories_dir: /var/lib/arvados/git
+
+ # This is a (bare) repository that stores commits used in jobs. When a job
+ # runs, the source commits are first fetched into this repository, then this
+ # repository is used to deploy to compute nodes. This should NOT be a
+ # subdirectory of {git_repositiories_dir}.
+ git_internal_dir: /var/lib/arvados/internal.git
+
+ # :none or :slurm_immediate
+ crunch_job_wrapper: :none
+
+ # username, or false = do not set uid when running jobs.
+ crunch_job_user: crunch
+
+ # The web service must be able to create/write this file, and
+ # crunch-job must be able to stat() it.
+ crunch_refresh_trigger: /tmp/crunch_refresh_trigger
+
+ # Path to /etc/dnsmasq.d, or false = do not update dnsmasq data.
+ dnsmasq_conf_dir: false
+
+ # Set to AMI id (ami-123456) to auto-start nodes. See app/models/node.rb
+ compute_node_ami: false
+ compute_node_ec2run_args: -g arvados-compute
+ compute_node_spot_bid: 0.11
+
+ compute_node_domain: false
+ compute_node_nameservers:
+ - 192.168.1.1
+ compute_node_ec2_tag_enable: false
+
+ accept_api_token: {}
+
+ new_users_are_active: false
+ admin_notifier_email_from: arvados@example.com
+ email_subject_prefix: "[ARVADOS] "
+ user_notifier_email_from: arvados@example.com
+
+ # Visitors to the API server will be redirected to the workbench
+ workbench_address: https://workbench.local:3001/
+
+ # The e-mail address of the user you would like to become marked as an admin
+ # user on their first login.
+ # In the default configuration, authentication happens through the Arvados SSO
+ # server, which uses openid against Google's servers, so in that case this
+ # should be an address associated with a Google account.
+ auto_admin_user: false
+
+ ## Set Time.zone default to the specified zone and make Active
+ ## Record auto-convert to this zone. Run "rake -D time" for a list
+ ## of tasks for finding time zone names. Default is UTC.
+ #time_zone: Central Time (US & Canada)
+
+ ## Default encoding used in templates for Ruby 1.9.
+ encoding: utf-8
+
+ # Enable the asset pipeline
+ assets.enabled: true
+
+ # Version of your assets, change this if you want to expire all your assets
+ assets.version: "1.0"
+
+ arvados_theme: default
# Activate observers that should always be running.
# config.active_record.observers = :cacher, :garbage_collector, :forum_observer
- # Set Time.zone default to the specified zone and make Active Record auto-convert to this zone.
- # Run "rake -D time" for a list of tasks for finding time zone names. Default is UTC.
- # config.time_zone = 'Central Time (US & Canada)'
-
# The default locale is :en and all translations from config/locales/*.rb,yml are auto loaded.
# config.i18n.load_path += Dir[Rails.root.join('my', 'locales', '*.{rb,yml}').to_s]
# config.i18n.default_locale = :de
- # Configure the default encoding used in templates for Ruby 1.9.
- config.encoding = "utf-8"
-
# Configure sensitive parameters which will be filtered from the log file.
config.filter_parameters += [:password]
-
- # Enable the asset pipeline
- config.assets.enabled = true
-
- # Version of your assets, change this if you want to expire all your assets
- config.assets.version = '1.0'
-
- config.force_ssl = true
-
- def config.uuid_prefix(x=nil)
- if x and @uuid_prefix
- raise "uuid_prefix was already set to #{@uuid_prefix}"
- end
- @uuid_prefix ||= Digest::MD5.hexdigest(x || `hostname`.strip).to_i(16).to_s(36)[-5..-1]
- end
- def config.uuid_prefix=(x)
- @uuid_prefix = x
- end
end
-
end
--- /dev/null
+# Copy this file to application.yml and edit to suit.
+#
+# Consult application.default.yml for the full list of configuration
+# settings.
+#
+# The order of precedence is:
+# 1. config/environments/{RAILS_ENV}.rb (deprecated)
+# 2. Section in application.yml corresponding to RAILS_ENV (e.g., development)
+# 3. Section in application.yml called "common"
+# 4. Section in application.default.yml corresponding to RAILS_ENV
+# 5. Section in application.default.yml called "common"
+
+development:
+
+production:
+ # At minimum, you need a nice long randomly generated secret_token here.
+ secret_token: ~
+
+ uuid_prefix: bogus
+
+ # compute_node_domain: example.org
+ # compute_node_nameservers:
+ # - 127.0.0.1
+ # - 192.168.1.1
+ #
+ # The version below is suitable for AWS.
+ # Uncomment and change <%# to <%= to use it.
+ # compute_node_nameservers: <%#
+ require 'net/http'
+ ['local', 'public'].collect do |iface|
+ Net::HTTP.get(URI("http://169.254.169.254/latest/meta-data/#{iface}-ipv4")).match(/^[\d\.]+$/)[0]
+ end << '172.16.0.23'
+ %>
+
+test:
+ uuid_prefix: zzzzz
+ secret_token: <%= rand(2**512).to_s(36) %>
+
+common:
+ #git_repositories_dir: /var/cache/git
+ #git_internal_dir: /var/cache/arvados/internal.git
development:
- adapter: mysql
- encoding: utf8
- database: arvados_development
- username: arvados
- password: ********
- host: localhost
+ adapter: sqlite3
+ database: db/arvados_development.sqlite3
test:
- adapter: mysql
+ adapter: postgresql
encoding: utf8
database: arvados_test
username: arvados
- password: ********
+ password: xxxxxxxx
host: localhost
production:
- adapter: mysql
+ adapter: postgresql
encoding: utf8
database: arvados_production
username: arvados
- password: ********
+ password: xxxxxxxx
host: localhost
# Initialize the rails application
Server::Application.initialize!
+begin
+ Rails.cache.clear
+rescue Errno::ENOENT => e
+ # Cache directory does not exist? Then cache is clear, proceed.
+ Rails.logger.warn "In Rails.cache.clear, ignoring #{e.inspect}"
+end
config.force_ssl = false
- config.git_repositories_dir = '/var/cache/git'
-
- config.crunch_job_wrapper = :none
- config.crunch_job_user = 'crunch' # if false, do not set uid when running jobs
-
- # The web service must be able to create/write this file, and
- # crunch-job must be able to stat() it.
- config.crunch_refresh_trigger = '/tmp/crunch_refresh_trigger'
-
- # config.dnsmasq_conf_dir = '/etc/dnsmasq.d'
-
- # config.compute_node_ami = 'ami-cbca41a2'
- # config.compute_node_ec2run_args = '-g arvados-compute'
- # config.compute_node_spot_bid = 0.11
-
- # config.compute_node_domain = `hostname --domain`.strip
-
- # config.compute_node_nameservers = ['1.2.3.4', '1.2.3.5']
- config.compute_node_nameservers = ['192.168.201.3']
-
- config.uuid_prefix('development@' + `hostname`.strip)
-
- # Authentication stub: hard code pre-approved API tokens.
- # config.accept_api_token = { rand(2**256).to_s(36) => true }
- config.accept_api_token = {}
-
- config.new_users_are_active = false
- config.admin_notifier_email_from = 'arvados@example.com'
- config.email_subject_prefix = '[ARVADOS] '
-
- # Visitors to the API server will be redirected to the workbench
- config.workbench_address = "http://localhost:3000/"
-
- # The e-mail address of the user you would like to become marked as an admin
- # user on their first login.
- # In the default configuration, authentication happens through the Arvados SSO
- # server, which uses openid against Google's servers, so in that case this
- # should be an address associated with a Google account.
- config.auto_admin_user = ''
end
# Send deprecation notices to registered listeners
config.active_support.deprecation = :notify
- config.git_repositories_dir = '/var/cache/git'
-
- config.crunch_job_wrapper = :slurm_immediate
- config.crunch_job_user = 'crunch' # if false, do not set uid when running jobs
-
- # The web service must be able to create/write this file, and
- # crunch-job must be able to stat() it.
- config.crunch_refresh_trigger = '/tmp/crunch_refresh_trigger'
-
- # config.dnsmasq_conf_dir = '/etc/dnsmasq.d'
-
- # config.compute_node_ami = 'ami-cbca41a2'
- # config.compute_node_ec2run_args = '-g arvados-compute'
- # config.compute_node_spot_bid = 0.11
-
- # config.compute_node_domain = `hostname --domain`.strip
-
- # config.compute_node_nameservers = ['1.2.3.4', '1.2.3.5']
- require 'net/http'
- config.compute_node_nameservers = ['local', 'public'].collect do |iface|
- Net::HTTP.get(URI("http://169.254.169.254/latest/meta-data/#{iface}-ipv4")).match(/^[\d\.]+$/)[0]
- end << '172.16.0.23'
-
- config.uuid_prefix = Digest::MD5.hexdigest('cfi-aws-0').to_i(16).to_s(36)[0..4] # '9ujm1'
-
- # Authentication stub: hard code pre-approved API tokens.
- # config.accept_api_token = { rand(2**256).to_s(36) => true }
- config.accept_api_token = {}
-
- config.new_users_are_active = false
- config.admin_notifier_email_from = 'arvados@example.com'
- config.email_subject_prefix = '[ARVADOS] '
-
- # Visitors to the API server will be redirected to the workbench
- config.workbench_address = "http://workbench." + `hostname`
-
- # The e-mail address of the user you would like to become marked as an admin
- # user on their first login.
- # In the default configuration, authentication happens through the Arvados SSO
- # server, which uses openid against Google's servers, so in that case this
- # should be an address associated with a Google account.
- config.auto_admin_user = ''
end
# Raise exception on mass assignment protection for Active Record models
config.active_record.mass_assignment_sanitizer = :strict
- config.git_repositories_dir = '/var/cache/git'
-
- config.crunch_job_wrapper = :slurm_immediate
- config.crunch_job_user = 'crunch' # if false, do not set uid when running jobs
-
- # The web service must be able to create/write this file, and
- # crunch-job must be able to stat() it.
- config.crunch_refresh_trigger = '/tmp/crunch_refresh_trigger_test'
-
- # config.dnsmasq_conf_dir = '/etc/dnsmasq.d'
-
- # config.compute_node_ami = 'ami-cbca41a2'
- # config.compute_node_ec2run_args = '-g arvados-compute'
- # config.compute_node_spot_bid = 0.11
- config.compute_node_ec2_tag_enable = false
-
- # config.compute_node_domain = `hostname --domain`.strip
-
# No need for SSL while testing
config.force_ssl = false
- # config.compute_node_nameservers = ['1.2.3.4', '1.2.3.5']
- config.compute_node_nameservers = [ "172.16.0.23" ]
-
- config.uuid_prefix = 'zzzzz'
-
- # Authentication stub: hard code pre-approved API tokens.
- # config.accept_api_token = { rand(2**256).to_s(36) => true }
- config.accept_api_token = {}
-
- config.new_users_are_active = false
- config.admin_notifier_email_from = 'arvados@example.com'
- config.email_subject_prefix = '[ARVADOS] '
-
- # Visitors to the API server will be redirected to the workbench
- config.workbench_address = "http://localhost:3000/"
end
APP_SECRET = rand(2**512).to_s(36) # CHANGE ME!
# Update your custom Omniauth provider URL here
-CUSTOM_PROVIDER_URL = 'http://auth.clinicalfuture.com'
+CUSTOM_PROVIDER_URL = 'http://localhost:3002'
Rails.application.config.middleware.use OmniAuth::Builder do
provider :josh_id, APP_ID, APP_SECRET, CUSTOM_PROVIDER_URL
--- /dev/null
+$application_config = {}
+
+%w(application.default application).each do |cfgfile|
+ path = "#{::Rails.root.to_s}/config/#{cfgfile}.yml"
+ if File.exists? path
+ yaml = ERB.new(IO.read path).result(binding)
+ confs = YAML.load(yaml)
+ $application_config.merge!(confs['common'] || {})
+ $application_config.merge!(confs[::Rails.env.to_s] || {})
+ end
+end
+
+Server::Application.configure do
+ nils = []
+ $application_config.each do |k, v|
+ # "foo.bar: baz" --> { config.foo.bar = baz }
+ cfg = config
+ ks = k.split '.'
+ k = ks.pop
+ ks.each do |kk|
+ cfg = cfg.send(kk)
+ end
+ if cfg.respond_to?(k.to_sym) and !cfg.send(k).nil?
+ # Config must have been set already in environments/*.rb.
+ #
+ # After config files have been migrated, this mechanism should
+ # be deprecated, then removed.
+ elsif v.nil?
+ # Config variables are not allowed to be nil. Make a "naughty"
+ # list, and present it below.
+ nils << k
+ else
+ cfg.send "#{k}=", v
+ end
+ end
+ if !nils.empty?
+ raise <<EOS
+Refusing to start in #{::Rails.env.to_s} mode with missing configuration.
+
+The following configuration settings must be specified in
+config/application.yml:
+* #{nils.join "\n* "}
+
+EOS
+ end
+end
Server::Application.routes.draw do
+ themes_for_rails
+
resources :humans
resources :traits
resources :repositories
namespace :arvados do
namespace :v1 do
- match '/schema' => 'schema#show'
match '/nodes/:uuid/ping' => 'nodes#ping', :as => :ping_node
match '/keep_disks/ping' => 'keep_disks#ping', :as => :ping_keep_disk
match '/links/from/:tail_uuid' => 'links#index', :as => :arvados_v1_links_from
post '/jobs/:uuid/cancel' => 'jobs#cancel'
match '/users/:uuid/event_stream' => 'users#event_stream'
post '/users/:uuid/activate' => 'users#activate'
+ post '/users/setup' => 'users#setup'
+ post '/users/:uuid/unsetup' => 'users#unsetup'
match '/virtual_machines/get_all_logins' => 'virtual_machines#get_all_logins'
match '/virtual_machines/:uuid/logins' => 'virtual_machines#logins'
post '/api_client_authorizations/create_system_auth' => 'api_client_authorizations#create_system_auth'
match '/login', :to => 'user_sessions#login'
match '/logout', :to => 'user_sessions#logout'
- match '/discovery/v1/apis/arvados/v1/rest', :to => 'arvados/v1/schema#discovery_rest_description'
+ match '/discovery/v1/apis/arvados/v1/rest', :to => 'arvados/v1/schema#index'
match '/static/login_failure', :to => 'static#login_failure', :as => :login_failure
--- /dev/null
+class AddNondeterministicColumnToJob < ActiveRecord::Migration
+ def up
+ add_column :jobs, :nondeterministic, :boolean
+ end
+
+ def down
+ remove_column :jobs, :nondeterministic
+ end
+end
--- /dev/null
+class SeparateRepositoryFromScriptVersion < ActiveRecord::Migration
+ include CurrentApiClient
+
+ def fixup pt
+ c = pt.components
+ c.each do |k, v|
+ commit_ish = v["script_version"]
+ if commit_ish.andand.index(':')
+ want_repo, commit_ish = commit_ish.split(':',2)
+ v[:repository] = want_repo
+ v[:script_version] = commit_ish
+ end
+ end
+ pt.save!
+ end
+
+ def up
+ act_as_system_user do
+ PipelineTemplate.all.each do |pt|
+ fixup pt
+ end
+ PipelineInstance.all.each do |pt|
+ fixup pt
+ end
+ end
+ end
+
+ def down
+ raise ActiveRecord::IrreversibleMigration
+ end
+end
--- /dev/null
+class AddRepositoryColumnToJob < ActiveRecord::Migration
+ def up
+ add_column :jobs, :repository, :string
+ end
+
+ def down
+ remove_column :jobs, :repository
+ end
+end
--- /dev/null
+class AddOutputIsPersistentToJob < ActiveRecord::Migration
+ def change
+ add_column :jobs, :output_is_persistent, :boolean, null: false, default: false
+ end
+end
--- /dev/null
+class RemoveKindColumns < ActiveRecord::Migration
+ include CurrentApiClient
+
+ def up
+ remove_column :links, :head_kind
+ remove_column :links, :tail_kind
+ remove_column :logs, :object_kind
+ end
+
+ def down
+ add_column :links, :head_kind, :string
+ add_column :links, :tail_kind, :string
+ add_column :logs, :object_kind, :string
+
+ act_as_system_user do
+ Link.all.each do |l|
+ l.head_kind = ArvadosModel::resource_class_for_uuid(l.head_uuid).kind if l.head_uuid
+ l.tail_kind = ArvadosModel::resource_class_for_uuid(l.tail_uuid).kind if l.tail_uuid
+ l.save
+ end
+ Log.all.each do |l|
+ l.object_kind = ArvadosModel::resource_class_for_uuid(l.object_uuid).kind if l.object_uuid
+ l.save
+ end
+ end
+ end
+end
--- /dev/null
+class AddSystemGroup < ActiveRecord::Migration
+ include CurrentApiClient
+
+ def up
+ # Make sure the system group exists.
+ system_group
+ end
+
+ def down
+ act_as_system_user do
+ system_group.destroy
+
+ # Destroy the automatically generated links giving system_group
+ # permission on all users.
+ Link.destroy_all(tail_uuid: system_group_uuid, head_kind: 'arvados#user')
+ end
+ end
+end
--- /dev/null
+class RenameLogInfoToProperties < ActiveRecord::Migration
+ def change
+ rename_column :logs, :info, :properties
+ end
+end
#
# It's strongly recommended to check this file into your version control system.
-ActiveRecord::Schema.define(:version => 20140129184311) do
+ActiveRecord::Schema.define(:version => 20140407184311) do
create_table "api_client_authorizations", :force => true do |t|
t.string "api_token", :null => false
create_table "collections", :force => true do |t|
t.string "locator"
t.string "owner_uuid"
- t.datetime "created_at", :null => false
+ t.datetime "created_at"
t.string "modified_by_client_uuid"
t.string "modified_by_user_uuid"
t.datetime "modified_at"
t.string "redundancy_confirmed_by_client_uuid"
t.datetime "redundancy_confirmed_at"
t.integer "redundancy_confirmed_as"
- t.datetime "updated_at", :null => false
+ t.datetime "updated_at"
t.string "uuid"
t.text "manifest_text"
end
t.string "repository_name"
t.string "sha1"
t.string "message"
- t.datetime "created_at", :null => false
- t.datetime "updated_at", :null => false
+ t.datetime "created_at"
+ t.datetime "updated_at"
end
add_index "commits", ["repository_name", "sha1"], :name => "index_commits_on_repository_name_and_sha1", :unique => true
t.string "modified_by_user_uuid"
t.datetime "modified_at"
t.text "properties"
- t.datetime "created_at", :null => false
- t.datetime "updated_at", :null => false
+ t.datetime "created_at"
+ t.datetime "updated_at"
end
add_index "humans", ["uuid"], :name => "index_humans_on_uuid", :unique => true
t.boolean "running"
t.boolean "success"
t.string "output"
- t.datetime "created_at", :null => false
- t.datetime "updated_at", :null => false
+ t.datetime "created_at"
+ t.datetime "updated_at"
t.string "priority"
t.string "is_locked_by_uuid"
t.string "log"
t.text "tasks_summary"
t.text "runtime_constraints"
+ t.boolean "nondeterministic"
+ t.string "repository"
+ t.boolean "output_is_persistent", :default => false, :null => false
end
add_index "jobs", ["created_at"], :name => "index_jobs_on_created_at"
create_table "links", :force => true do |t|
t.string "uuid"
t.string "owner_uuid"
- t.datetime "created_at", :null => false
+ t.datetime "created_at"
t.string "modified_by_client_uuid"
t.string "modified_by_user_uuid"
t.datetime "modified_at"
t.string "tail_uuid"
- t.string "tail_kind"
t.string "link_class"
t.string "name"
t.string "head_uuid"
t.text "properties"
- t.datetime "updated_at", :null => false
+ t.datetime "updated_at"
t.string "head_kind"
+ t.string "tail_kind"
end
add_index "links", ["created_at"], :name => "index_links_on_created_at"
- add_index "links", ["head_kind"], :name => "index_links_on_head_kind"
add_index "links", ["head_uuid"], :name => "index_links_on_head_uuid"
add_index "links", ["modified_at"], :name => "index_links_on_modified_at"
- add_index "links", ["tail_kind"], :name => "index_links_on_tail_kind"
add_index "links", ["tail_uuid"], :name => "index_links_on_tail_uuid"
add_index "links", ["uuid"], :name => "index_links_on_uuid", :unique => true
t.string "owner_uuid"
t.string "modified_by_client_uuid"
t.string "modified_by_user_uuid"
- t.string "object_kind"
t.string "object_uuid"
t.datetime "event_at"
t.string "event_type"
t.text "summary"
- t.text "info"
+ t.text "properties"
t.datetime "created_at", :null => false
t.datetime "updated_at", :null => false
t.datetime "modified_at"
add_index "logs", ["event_at"], :name => "index_logs_on_event_at"
add_index "logs", ["event_type"], :name => "index_logs_on_event_type"
add_index "logs", ["modified_at"], :name => "index_logs_on_modified_at"
- add_index "logs", ["object_kind"], :name => "index_logs_on_object_kind"
add_index "logs", ["object_uuid"], :name => "index_logs_on_object_uuid"
add_index "logs", ["summary"], :name => "index_logs_on_summary"
add_index "logs", ["uuid"], :name => "index_logs_on_uuid", :unique => true
create_table "pipeline_instances", :force => true do |t|
t.string "uuid"
t.string "owner_uuid"
- t.datetime "created_at", :null => false
+ t.datetime "created_at"
t.string "modified_by_client_uuid"
t.string "modified_by_user_uuid"
t.datetime "modified_at"
t.text "components"
t.boolean "success"
t.boolean "active", :default => false
- t.datetime "updated_at", :null => false
+ t.datetime "updated_at"
t.text "properties"
end
-# This file should contain all the record creation needed to seed the database with its default values.
-# The data can then be loaded with the rake db:seed (or created alongside the db with db:setup).
+# This file seeds the database with initial/default values.
#
-# Examples:
-#
-# cities = City.create([{ :name => 'Chicago' }, { :name => 'Copenhagen' }])
-# Mayor.create(:name => 'Emanuel', :city => cities.first)
+# It is invoked by `rake db:seed` and `rake db:setup`.
+
+# These two methods would create the system user and group objects on
+# demand later anyway, but it's better form to create them up front.
+include CurrentApiClient
+system_user
+system_group
'000000000000000'].join('-')
end
+ def system_group_uuid
+ [Server::Application.config.uuid_prefix,
+ Group.uuid_prefix,
+ '000000000000000'].join('-')
+ end
+
def system_user
if not $system_user
real_current_user = Thread.current[:user]
$system_user
end
+ def system_group
+ if not $system_group
+ act_as_system_user do
+ ActiveRecord::Base.transaction do
+ $system_group = Group.
+ where(uuid: system_group_uuid).first_or_create do |g|
+ g.update_attributes(name: "System group",
+ description: "System group")
+ User.all.collect(&:uuid).each do |user_uuid|
+ Link.create(link_class: 'permission',
+ name: 'can_manage',
+ tail_kind: 'arvados#group',
+ tail_uuid: system_group_uuid,
+ head_kind: 'arvados#user',
+ head_uuid: user_uuid)
+ end
+ end
+ end
+ end
+ end
+ $system_group
+ end
+
def act_as_system_user
if block_given?
user_was = Thread.current[:user]
end
module ClassMethods
+ def kind
+ 'arvados#' + self.to_s.camelcase(:lower)
+ end
end
def kind
- 'arvados#' + self.class.to_s.camelcase(:lower)
+ self.class.kind
end
def etag
--- /dev/null
+namespace :config do
+ desc 'Ensure site configuration has all required settings'
+ task check: :environment do
+ $application_config.sort.each do |k, v|
+ if ENV.has_key?('QUIET') then
+ # Make sure we still check for the variable to exist
+ eval("Rails.configuration.#{k}")
+ else
+ if /(password|secret)/.match(k) then
+ # Make sure we still check for the variable to exist, but don't print the value
+ eval("Rails.configuration.#{k}")
+ $stderr.puts "%-32s %s" % [k, '*********']
+ else
+ $stderr.puts "%-32s %s" % [k, eval("Rails.configuration.#{k}")]
+ end
+ end
+ end
+ end
+end
def refresh_todo
@todo = Job.queue
+ @todo_pipelines = PipelineInstance.queue
end
def sinfo
raise "No CRUNCH_JOB_BIN env var, and crunch-job not in path."
end
+ require 'shellwords'
+
+ arvados_internal = Rails.configuration.git_internal_dir
+ if not File.exists? arvados_internal
+ $stderr.puts `mkdir -p #{arvados_internal.shellescape} && cd #{arvados_internal.shellescape} && git init --bare`
+ end
+
+ src_repo = File.join(Rails.configuration.git_repositories_dir, job.repository + '.git')
+ src_repo = File.join(Rails.configuration.git_repositories_dir, job.repository, '.git') unless File.exists? src_repo
+
+ unless src_repo
+ $stderr.puts "dispatch: #{File.join Rails.configuration.git_repositories_dir, job.repository} doesn't exist"
+ sleep 1
+ untake(job)
+ next
+ end
+
+ $stderr.puts `cd #{arvados_internal.shellescape} && git fetch --no-tags #{src_repo.shellescape} && git tag #{job.uuid.shellescape} #{job.script_version.shellescape}`
+
cmd_args << crunch_job_bin
cmd_args << '--job-api-token'
cmd_args << job_auth.api_token
cmd_args << '--job'
cmd_args << job.uuid
-
- commit = Commit.where(sha1: job.script_version).first
- if commit
- cmd_args << '--git-dir'
- if File.exists?(File.
- join(Rails.configuration.git_repositories_dir,
- commit.repository_name + '.git'))
- cmd_args << File.
- join(Rails.configuration.git_repositories_dir,
- commit.repository_name + '.git')
- else
- cmd_args << File.
- join(Rails.configuration.git_repositories_dir,
- commit.repository_name, '.git')
- end
- end
+ cmd_args << '--git-dir'
+ cmd_args << arvados_internal
$stderr.puts "dispatch: #{cmd_args.join ' '}"
job_done = j_done[:job]
$stderr.puts "dispatch: child #{pid_done} exit"
$stderr.puts "dispatch: job #{job_done.uuid} end"
- $redis.publish job_done.uuid, "end"
# Ensure every last drop of stdout and stderr is consumed
read_pipes
# Wait the thread
j_done[:wait_thr].value
+ jobrecord = Job.find_by_uuid(job_done.uuid)
+ jobrecord.running = false
+ jobrecord.finished_at ||= Time.now
+ # Don't set 'jobrecord.success = false' because if the job failed to run due to an
+ # issue with crunch-job or slurm, we want the job to stay in the queue.
+ jobrecord.save!
+
# Invalidate the per-job auth token
j_done[:job_auth].update_attributes expires_at: Time.now
+ $redis.publish job_done.uuid, "end"
+
@running.delete job_done.uuid
end
+ def update_pipelines
+ expire_tokens = @pipe_auth_tokens.dup
+ @todo_pipelines.each do |p|
+ pipe_auth = (@pipe_auth_tokens[p.uuid] ||= ApiClientAuthorization.
+ create(user: User.where('uuid=?', p.modified_by_user_uuid).first,
+ api_client_id: 0))
+ puts `export ARVADOS_API_TOKEN=#{pipe_auth.api_token} && arv-run-pipeline-instance --run-here --no-wait --instance #{p.uuid}`
+ expire_tokens.delete p.uuid
+ end
+
+ expire_tokens.each do |k, v|
+ v.update_attributes expires_at: Time.now
+ @pipe_auth_tokens.delete k
+ end
+ end
+
def run
act_as_system_user
@running ||= {}
+ @pipe_auth_tokens ||= { }
$stderr.puts "dispatch: ready"
while !$signal[:term] or @running.size > 0
read_pipes
unless @todo.empty? or did_recently(:start_jobs, 1.0) or $signal[:term]
start_jobs
end
+ unless (@todo_pipelines.empty? and @pipe_auth_tokens.empty?) or did_recently(:update_pipelines, 5.0)
+ update_pipelines
+ end
end
reap_children
select(@running.values.collect { |j| [j[:stdout], j[:stderr]] }.flatten,
:config => File.expand_path("config.ru"),
:SSLEnable => true,
:SSLVerifyClient => OpenSSL::SSL::VERIFY_NONE,
- :SSLPrivateKey => OpenSSL::PKey::RSA.new(
- File.open("config/api.clinicalfuture.com.key.pem").read),
- :SSLCertificate => OpenSSL::X509::Certificate.new(
- File.open("config/api.clinicalfuture.com.crt.pem").read),
- :SSLCertName => [["CN", WEBrick::Utils::getservername]]
+ :SSLCertName => [["CN", "#{WEBrick::Utils::getservername} #{Time.now().to_s}"]]
})
end
end
-end
+end
######### /SSL
--- /dev/null
+#!/usr/bin/env ruby
+
+abort 'Error: Ruby >= 1.9.3 required.' if RUBY_VERSION < '1.9.3'
+
+require 'logger'
+require 'trollop'
+
+log = Logger.new STDERR
+log.progname = $0.split('/').last
+
+opts = Trollop::options do
+ banner ''
+ banner "Usage: #{log.progname} " +
+ "{user_uuid_or_email} {user_and_repo_name} {vm_uuid}"
+ banner ''
+ opt :debug, <<-eos
+Show debug messages.
+ eos
+ opt :openid_prefix, <<-eos, default: 'https://www.google.com/accounts/o8/id'
+If creating a new user record, require authentication from an OpenID \
+with this OpenID prefix *and* a matching email address in order to \
+claim the account.
+ eos
+ opt :send_notification_email, <<-eos, default: 'true'
+Send notification email after successfully setting up the user.
+ eos
+end
+
+log.level = (ENV['DEBUG'] || opts.debug) ? Logger::DEBUG : Logger::WARN
+
+if ARGV.count != 3
+ Trollop::die "required arguments are missing"
+end
+
+user_arg, user_repo_name, vm_uuid = ARGV
+
+require 'arvados'
+arv = Arvados.new(api_version: 'v1')
+
+# Look up the given user by uuid or, failing that, email address.
+begin
+ found_user = arv.user.get(uuid: user_arg)
+rescue Arvados::TransactionFailedError
+ found = arv.user.list(where: {email: user_arg})[:items]
+
+ if found.count == 0
+ if !user_arg.match(/\w\@\w+\.\w+/)
+ abort "About to create new user, but #{user_arg.inspect} " +
+ "does not look like an email address. Stop."
+ end
+ elsif found.count != 1
+ abort "Found #{found.count} users with email. Stop."
+ else
+ found_user = found.first
+ end
+end
+
+# Invoke user setup method
+if (found_user)
+ user = arv.user.setup uuid: found_user[:uuid], repo_name: user_repo_name,
+ vm_uuid: vm_uuid, openid_prefix: opts.openid_prefix,
+ send_notification_email: opts.send_notification_email
+else
+ user = arv.user.setup user: {email: user_arg}, repo_name: user_repo_name,
+ vm_uuid: vm_uuid, openid_prefix: opts.openid_prefix,
+ send_notification_email: opts.send_notification_email
+end
+
+log.info {"user uuid: " + user[:uuid]}
+
+puts user.inspect
api_token: 1a9ffdcga2o7cw8q12dndskomgs1ygli3ns9k2o9hgzgmktc78
expires_at: 2038-01-01 00:00:00
+miniadmin:
+ api_client: untrusted
+ user: miniadmin
+ api_token: 2zb2y9pw3e70270te7oe3ewaantea3adyxjascvkz0zob7q7xb
+ expires_at: 2038-01-01 00:00:00
+
+rominiadmin:
+ api_client: untrusted
+ user: rominiadmin
+ api_token: 5tsb2pc3zlatn1ortl98s2tqsehpby88wmmnzmpsjmzwa6payh
+ expires_at: 2038-01-01 00:00:00
+
active:
api_client: untrusted
user: active
user: active
api_token: 5hpni7izokzcatku2896xxwqdbt5ptomn04r6auc7fohnli82v
expires_at: 1970-01-01 00:00:00
+
+valid_token_deleted_user:
+ api_client: trusted_workbench
+ user_id: 1234567
+ api_token: tewfa58099sndckyqhlgd37za6e47o6h03r9l1vpll23hudm8b
+ expires_at: 2038-01-01 00:00:00
user_agreement:
- uuid: b519d9cb706a29fc7ea24dbea2f05851
+ uuid: b519d9cb706a29fc7ea24dbea2f05851+249025
owner_uuid: qr1hi-tpzed-tpj2ff66551eyym
created_at: 2013-12-26T19:22:54Z
modified_by_client_uuid: zzzzz-ozdt8-brczlopd8u8d0jr
name: Private
description: Private Group
+private_and_can_read_foofile:
+ uuid: zzzzz-j7d0g-22xp1wpjul508rk
+ owner_uuid: zzzzz-tpzed-xurymjxw79nv3jz
+ name: Private and Can Read Foofile
+ description: Another Private Group
+
system_owned_group:
uuid: zzzzz-j7d0g-8ulrifv67tve5sx
owner_uuid: zzzzz-tpzed-000000000000000
name: System Private
description: System-owned Group
+system_group:
+ uuid: zzzzz-j7d0g-000000000000000
+ owner_uuid: zzzzz-tpzed-000000000000000
+ name: System Private
+ description: System-owned Group
+
empty_lonely_group:
uuid: zzzzz-j7d0g-jtp06ulmvsezgyu
owner_uuid: zzzzz-tpzed-000000000000000
uuid: zzzzz-j7d0g-fffffffffffffff
owner_uuid: zzzzz-tpzed-d9tiejq69daie8f
name: All users
+
+testusergroup_admins:
+ uuid: zzzzz-j7d0g-48foin4vonvc2at
+ owner_uuid: zzzzz-tpzed-000000000000000
+ name: Administrators of a subset of users
cancelled_by_client_uuid: ~
started_at: <%= 3.minute.ago.to_s(:db) %>
finished_at: ~
+ script_version: 1de84a854e2b440dc53bf42f8548afa4c17da332
running: true
success: ~
output: ~
cancelled_by_client_uuid: zzzzz-ozdt8-obw7foaks3qjyej
started_at: <%= 3.minute.ago.to_s(:db) %>
finished_at: ~
+ script_version: 1de84a854e2b440dc53bf42f8548afa4c17da332
running: true
success: ~
output: ~
running: 0
done: 1
runtime_constraints: {}
+
+previous_job_run:
+ uuid: zzzzz-8i9sb-cjs4pklxxjykqqq
+ owner_uuid: zzzzz-tpzed-xurymjxw79nv3jz
+ script: hash
+ script_version: 4fe459abe02d9b365932b8f5dc419439ab4e2577
+ script_parameters:
+ input: fa7aeb5140e2848d39b416daeef4ffc5+45
+ an_integer: "1"
+ success: true
+
+nondeterminisic_job_run:
+ uuid: zzzzz-8i9sb-cjs4pklxxjykyyy
+ owner_uuid: zzzzz-tpzed-xurymjxw79nv3jz
+ script: hash2
+ script_version: 4fe459abe02d9b365932b8f5dc419439ab4e2577
+ script_parameters:
+ input: fa7aeb5140e2848d39b416daeef4ffc5+45
+ an_integer: "1"
+ success: true
+ nondeterministic: true
\ No newline at end of file
modified_by_user_uuid: zzzzz-tpzed-d9tiejq69daie8f
modified_at: 2013-12-26T19:52:21Z
updated_at: 2013-12-26T19:52:21Z
- tail_kind: arvados#user
tail_uuid: zzzzz-tpzed-000000000000000
link_class: signature
name: require
- head_kind: arvados#collection
- head_uuid: b519d9cb706a29fc7ea24dbea2f05851
+ head_uuid: b519d9cb706a29fc7ea24dbea2f05851+249025
properties: {}
user_agreement_readable:
modified_by_user_uuid: zzzzz-tpzed-d9tiejq69daie8f
modified_at: 2014-01-24 20:42:26 -0800
updated_at: 2014-01-24 20:42:26 -0800
- tail_kind: arvados#group
tail_uuid: zzzzz-j7d0g-fffffffffffffff
link_class: permission
name: can_read
- head_kind: arvados#collection
- head_uuid: b519d9cb706a29fc7ea24dbea2f05851
+ head_uuid: b519d9cb706a29fc7ea24dbea2f05851+249025
properties: {}
active_user_member_of_all_users_group:
modified_by_user_uuid: zzzzz-tpzed-d9tiejq69daie8f
modified_at: 2014-01-24 20:42:26 -0800
updated_at: 2014-01-24 20:42:26 -0800
- tail_kind: arvados#user
tail_uuid: zzzzz-tpzed-xurymjxw79nv3jz
link_class: permission
name: can_read
- head_kind: arvados#group
head_uuid: zzzzz-j7d0g-fffffffffffffff
properties: {}
modified_by_user_uuid: zzzzz-tpzed-000000000000000
modified_at: 2014-02-03 15:42:26 -0800
updated_at: 2014-02-03 15:42:26 -0800
- tail_kind: arvados#user
tail_uuid: zzzzz-tpzed-xurymjxw79nv3jz
link_class: permission
name: can_manage
- head_kind: arvados#group
head_uuid: zzzzz-j7d0g-8ulrifv67tve5sx
properties: {}
modified_by_user_uuid: zzzzz-tpzed-xurymjxw79nv3jz
modified_at: 2013-12-26T20:52:21Z
updated_at: 2013-12-26T20:52:21Z
- tail_kind: arvados#user
tail_uuid: zzzzz-tpzed-xurymjxw79nv3jz
link_class: signature
name: click
- head_kind: arvados#collection
- head_uuid: b519d9cb706a29fc7ea24dbea2f05851
+ head_uuid: b519d9cb706a29fc7ea24dbea2f05851+249025
properties: {}
user_agreement_signed_by_inactive:
modified_by_user_uuid: zzzzz-tpzed-7sg468ezxwnodxs
modified_at: 2013-12-26T20:52:21Z
updated_at: 2013-12-26T20:52:21Z
- tail_kind: arvados#user
tail_uuid: zzzzz-tpzed-7sg468ezxwnodxs
link_class: signature
name: click
- head_kind: arvados#collection
- head_uuid: b519d9cb706a29fc7ea24dbea2f05851
+ head_uuid: b519d9cb706a29fc7ea24dbea2f05851+249025
properties: {}
spectator_user_member_of_all_users_group:
modified_by_user_uuid: zzzzz-tpzed-d9tiejq69daie8f
modified_at: 2014-01-24 20:42:26 -0800
updated_at: 2014-01-24 20:42:26 -0800
- tail_kind: arvados#user
tail_uuid: zzzzz-tpzed-l1s2piq4t4mps8r
link_class: permission
name: can_read
- head_kind: arvados#group
head_uuid: zzzzz-j7d0g-fffffffffffffff
properties: {}
modified_by_user_uuid: zzzzz-tpzed-7sg468ezxwnodxs
modified_at: 2013-12-26T20:52:21Z
updated_at: 2013-12-26T20:52:21Z
- tail_kind: arvados#user
tail_uuid: zzzzz-tpzed-x9kqpd79egh49c7
link_class: permission
name: can_read
- head_kind: arvados#group
head_uuid: zzzzz-j7d0g-fffffffffffffff
properties: {}
modified_by_user_uuid: zzzzz-tpzed-7sg468ezxwnodxs
modified_at: 2013-12-26T20:52:21Z
updated_at: 2013-12-26T20:52:21Z
- tail_kind: arvados#user
tail_uuid: zzzzz-tpzed-7sg468ezxwnodxs
link_class: permission
name: can_read
- head_kind: arvados#group
head_uuid: zzzzz-j7d0g-fffffffffffffff
properties: {}
modified_by_user_uuid: zzzzz-tpzed-000000000000000
modified_at: 2014-01-24 20:42:26 -0800
updated_at: 2014-01-24 20:42:26 -0800
- tail_kind: arvados#user
tail_uuid: zzzzz-tpzed-xurymjxw79nv3jz
link_class: permission
name: can_read
- head_kind: arvados#collection
+ head_uuid: 1f4b0bc7583c2a7f9102c395f4ffc5e3+45
+ properties: {}
+
+foo_file_readable_by_active_duplicate_permission:
+ uuid: zzzzz-o0j2j-2qlmhgothiur55r
+ owner_uuid: zzzzz-tpzed-000000000000000
+ created_at: 2014-01-24 20:42:26 -0800
+ modified_by_client_uuid: zzzzz-ozdt8-000000000000000
+ modified_by_user_uuid: zzzzz-tpzed-000000000000000
+ modified_at: 2014-01-24 20:42:26 -0800
+ updated_at: 2014-01-24 20:42:26 -0800
+ tail_uuid: zzzzz-tpzed-xurymjxw79nv3jz
+ link_class: permission
+ name: can_read
+ head_uuid: 1f4b0bc7583c2a7f9102c395f4ffc5e3+45
+ properties: {}
+
+foo_file_readable_by_active_redundant_permission_via_private_group:
+ uuid: zzzzz-o0j2j-5s8ry7sn6bwxb7w
+ owner_uuid: zzzzz-tpzed-000000000000000
+ created_at: 2014-01-24 20:42:26 -0800
+ modified_by_client_uuid: zzzzz-ozdt8-000000000000000
+ modified_by_user_uuid: zzzzz-tpzed-000000000000000
+ modified_at: 2014-01-24 20:42:26 -0800
+ updated_at: 2014-01-24 20:42:26 -0800
+ tail_uuid: zzzzz-j7d0g-22xp1wpjul508rk
+ link_class: permission
+ name: can_read
head_uuid: 1f4b0bc7583c2a7f9102c395f4ffc5e3+45
properties: {}
modified_by_user_uuid: zzzzz-tpzed-000000000000000
modified_at: 2014-01-24 20:42:26 -0800
updated_at: 2014-01-24 20:42:26 -0800
- tail_kind: arvados#user
tail_uuid: zzzzz-tpzed-xurymjxw79nv3jz
link_class: permission
name: can_read
- head_kind: arvados#collection
head_uuid: fa7aeb5140e2848d39b416daeef4ffc5+45
properties: {}
modified_by_user_uuid: zzzzz-tpzed-000000000000000
modified_at: 2014-01-24 20:42:26 -0800
updated_at: 2014-01-24 20:42:26 -0800
- tail_kind: arvados#user
tail_uuid: zzzzz-tpzed-l1s2piq4t4mps8r
link_class: permission
name: can_read
- head_kind: arvados#collection
head_uuid: fa7aeb5140e2848d39b416daeef4ffc5+45
properties: {}
modified_by_user_uuid: zzzzz-tpzed-000000000000000
modified_at: 2014-01-24 20:42:26 -0800
updated_at: 2014-01-24 20:42:26 -0800
- tail_kind: arvados#group
tail_uuid: zzzzz-j7d0g-fffffffffffffff
link_class: permission
name: can_read
- head_kind: arvados#collection
head_uuid: ea10d51bcf88862dbcc36eb292017dfd+45
properties: {}
modified_by_user_uuid: zzzzz-tpzed-000000000000000
modified_at: 2014-01-24 20:42:26 -0800
updated_at: 2014-01-24 20:42:26 -0800
- tail_kind: arvados#user
tail_uuid: zzzzz-tpzed-l1s2piq4t4mps8r
link_class: permission
name: can_read
- head_kind: arvados#job
head_uuid: zzzzz-8i9sb-cjs4pklxxjykyuq
properties: {}
+foo_repository_readable_by_spectator:
+ uuid: zzzzz-o0j2j-cpy7p41hpk5xxx
+ owner_uuid: zzzzz-tpzed-000000000000000
+ created_at: 2014-01-24 20:42:26 -0800
+ modified_by_client_uuid: zzzzz-ozdt8-brczlopd8u8d0jr
+ modified_by_user_uuid: zzzzz-tpzed-000000000000000
+ modified_at: 2014-01-24 20:42:26 -0800
+ updated_at: 2014-01-24 20:42:26 -0800
+ tail_uuid: zzzzz-tpzed-l1s2piq4t4mps8r
+ link_class: permission
+ name: can_read
+ head_uuid: zzzzz-2x53u-382brsig8rp3666
+ properties: {}
+
+miniadmin_user_is_a_testusergroup_admin:
+ uuid: zzzzz-o0j2j-38vvkciz7qc12j9
+ owner_uuid: zzzzz-tpzed-000000000000000
+ created_at: 2014-04-01 13:53:33 -0400
+ modified_by_client_uuid: zzzzz-ozdt8-brczlopd8u8d0jr
+ modified_by_user_uuid: zzzzz-tpzed-000000000000000
+ modified_at: 2014-04-01 13:53:33 -0400
+ updated_at: 2014-04-01 13:53:33 -0400
+ tail_uuid: zzzzz-tpzed-2bg9x0oeydcw5hm
+ link_class: permission
+ name: can_manage
+ head_uuid: zzzzz-j7d0g-48foin4vonvc2at
+ properties: {}
+
+rominiadmin_user_is_a_testusergroup_admin:
+ uuid: zzzzz-o0j2j-6b0hz5hr107mc90
+ owner_uuid: zzzzz-tpzed-000000000000000
+ created_at: 2014-04-01 13:53:33 -0400
+ modified_by_client_uuid: zzzzz-ozdt8-brczlopd8u8d0jr
+ modified_by_user_uuid: zzzzz-tpzed-000000000000000
+ modified_at: 2014-04-01 13:53:33 -0400
+ updated_at: 2014-04-01 13:53:33 -0400
+ tail_uuid: zzzzz-tpzed-4hvxm4n25emegis
+ link_class: permission
+ name: can_read
+ head_uuid: zzzzz-j7d0g-48foin4vonvc2at
+ properties: {}
+
+testusergroup_can_manage_active_user:
+ uuid: zzzzz-o0j2j-2vaqhxz6hsf4k1d
+ owner_uuid: zzzzz-tpzed-000000000000000
+ created_at: 2014-04-01 13:56:10 -0400
+ modified_by_client_uuid: zzzzz-ozdt8-brczlopd8u8d0jr
+ modified_by_user_uuid: zzzzz-tpzed-000000000000000
+ modified_at: 2014-04-01 13:56:10 -0400
+ updated_at: 2014-04-01 13:56:10 -0400
+ tail_uuid: zzzzz-j7d0g-48foin4vonvc2at
+ link_class: permission
+ name: can_manage
+ head_uuid: zzzzz-tpzed-xurymjxw79nv3jz
+ properties: {}
--- /dev/null
+log1:
+ uuid: zzzzz-xxxxx-pshmckwoma9plh7
+ object_uuid: zzzzz-tpzed-l1s2piq4t4mps8r
\ No newline at end of file
--- /dev/null
+foo:
+ uuid: zzzzz-2x53u-382brsig8rp3666
+ owner_uuid: zzzzz-tpzed-xurymjxw79nv3jz
+ name: foo
--- /dev/null
+owned_by_active_user:
+ uuid: zzzzz-2x53u-3zx463qyo0k4xrn
+ owner_uuid: zzzzz-tpzed-xurymjxw79nv3jz
+
+owned_by_private_group:
+ uuid: zzzzz-2x53u-5m3qwg45g3nlpu6
+ owner_uuid: zzzzz-j7d0g-rew6elm53kancon
+
+owned_by_spectator:
+ uuid: zzzzz-2x53u-3b0xxwzlbzxq5yr
+ owner_uuid: zzzzz-tpzed-l1s2piq4t4mps8r
is_admin: true
prefs: {}
+miniadmin:
+ uuid: zzzzz-tpzed-2bg9x0oeydcw5hm
+ email: miniadmin@arvados.local
+ first_name: TestCase
+ last_name: User Group Administrator
+ identity_url: https://miniadmin.openid.local
+ is_active: true
+ is_admin: false
+ prefs: {}
+
+rominiadmin:
+ uuid: zzzzz-tpzed-4hvxm4n25emegis
+ email: rominiadmin@arvados.local
+ first_name: TestCase
+ last_name: Read-Only User Group Administrator
+ identity_url: https://rominiadmin.openid.local
+ is_active: true
+ is_admin: false
+ prefs: {}
+
active:
uuid: zzzzz-tpzed-xurymjxw79nv3jz
email: active-user@arvados.local
uuid: zzzzz-2x53u-382brsig8rp3064
owner_uuid: zzzzz-tpzed-d9tiejq69daie8f
hostname: testvm.shell
+
+testvm2:
+ uuid: zzzzz-2x53u-382brsig8rp3065
+ owner_uuid: zzzzz-tpzed-d9tiejq69daie8f
+ hostname: testvm2.shell
assert_not_nil assigns(:objects)
end
+ [0,1,2].each do |limit|
+ test "get index with limit=#{limit}" do
+ authorize_with :active
+ get :index, limit: limit
+ assert_response :success
+ assert_equal limit, assigns(:objects).count
+ resp = JSON.parse(@response.body)
+ assert_equal limit, resp['limit']
+ end
+ end
+
+ test "items.count == items_available" do
+ authorize_with :active
+ get :index, limit: 100000
+ assert_response :success
+ resp = JSON.parse(@response.body)
+ assert_equal resp['items_available'], assigns(:objects).length
+ assert_equal resp['items_available'], resp['items'].count
+ unique_uuids = resp['items'].collect { |i| i['uuid'] }.compact.uniq
+ assert_equal unique_uuids.count, resp['items'].count
+ end
+
+ test "get index with limit=2 offset=99999" do
+ # Assume there are not that many test fixtures.
+ authorize_with :active
+ get :index, limit: 2, offset: 99999
+ assert_response :success
+ assert_equal 0, assigns(:objects).count
+ resp = JSON.parse(@response.body)
+ assert_equal 2, resp['limit']
+ assert_equal 99999, resp['offset']
+ end
+
test "should create" do
authorize_with :active
test_collection = {
assert_nil resp['1f4b0bc7583c2a7f9102c395f4ffc5e3+45'] # foo
end
+ test "search collections with 'any' operator" do
+ authorize_with :active
+ get :index, {
+ where: { any: ['contains', '7f9102c395f4ffc5e3'] }
+ }
+ assert_response :success
+ found = assigns(:objects).collect(&:uuid)
+ assert_equal 1, found.count
+ assert_equal true, !!found.index('1f4b0bc7583c2a7f9102c395f4ffc5e3+45')
+ end
+
end
--- /dev/null
+require 'test_helper'
+load 'test/functional/arvados/v1/git_setup.rb'
+
+class Arvados::V1::CommitsControllerTest < ActionController::TestCase
+ fixtures :repositories, :users
+
+ # See git_setup.rb for the commit log for test.git.tar
+ include GitSetup
+
+ test "test_find_commit_range" do
+ authorize_with :active
+
+ # single
+ a = Commit.find_commit_range(users(:active), nil, nil, '31ce37fe365b3dc204300a3e4c396ad333ed0556', nil)
+ assert_equal ['31ce37fe365b3dc204300a3e4c396ad333ed0556'], a
+
+ #test "test_branch1" do
+ a = Commit.find_commit_range(users(:active), nil, nil, 'master', nil)
+ assert_equal ['077ba2ad3ea24a929091a9e6ce545c93199b8e57'], a
+
+ #test "test_branch2" do
+ a = Commit.find_commit_range(users(:active), 'foo', nil, 'b1', nil)
+ assert_equal ['1de84a854e2b440dc53bf42f8548afa4c17da332'], a
+
+ #test "test_branch3" do
+ a = Commit.find_commit_range(users(:active), 'foo', nil, 'HEAD', nil)
+ assert_equal ['1de84a854e2b440dc53bf42f8548afa4c17da332'], a
+
+ #test "test_single_revision_repo" do
+ a = Commit.find_commit_range(users(:active), "foo", nil, '31ce37fe365b3dc204300a3e4c396ad333ed0556', nil)
+ assert_equal ['31ce37fe365b3dc204300a3e4c396ad333ed0556'], a
+ a = Commit.find_commit_range(users(:active), "bar", nil, '31ce37fe365b3dc204300a3e4c396ad333ed0556', nil)
+ assert_equal nil, a
+
+ #test "test_multi_revision" do
+ a = Commit.find_commit_range(users(:active), nil, '31ce37fe365b3dc204300a3e4c396ad333ed0556', '077ba2ad3ea24a929091a9e6ce545c93199b8e57', nil)
+ assert_equal ['077ba2ad3ea24a929091a9e6ce545c93199b8e57', '4fe459abe02d9b365932b8f5dc419439ab4e2577', '31ce37fe365b3dc204300a3e4c396ad333ed0556'], a
+
+ #test "test_tag" do
+ a = Commit.find_commit_range(users(:active), nil, 'tag1', 'master', nil)
+ assert_equal ['077ba2ad3ea24a929091a9e6ce545c93199b8e57', '4fe459abe02d9b365932b8f5dc419439ab4e2577'], a
+
+ #test "test_multi_revision_exclude" do
+ a = Commit.find_commit_range(users(:active), nil, '31ce37fe365b3dc204300a3e4c396ad333ed0556', '077ba2ad3ea24a929091a9e6ce545c93199b8e57', ['4fe459abe02d9b365932b8f5dc419439ab4e2577'])
+ assert_equal ['077ba2ad3ea24a929091a9e6ce545c93199b8e57', '31ce37fe365b3dc204300a3e4c396ad333ed0556'], a
+
+ #test "test_multi_revision_tagged_exclude" do
+ a = Commit.find_commit_range(users(:active), nil, '31ce37fe365b3dc204300a3e4c396ad333ed0556', '077ba2ad3ea24a929091a9e6ce545c93199b8e57', ['tag1'])
+ assert_equal ['077ba2ad3ea24a929091a9e6ce545c93199b8e57', '31ce37fe365b3dc204300a3e4c396ad333ed0556'], a
+
+ Dir.mktmpdir do |touchdir|
+ # invalid input to maximum
+ a = Commit.find_commit_range(users(:active), nil, nil, "31ce37fe365b3dc204300a3e4c396ad333ed0556 ; touch #{touchdir}/uh_oh", nil)
+ assert !File.exists?("#{touchdir}/uh_oh"), "#{touchdir}/uh_oh should not exist, 'maximum' parameter of find_commit_range is exploitable"
+ assert_equal nil, a
+
+ # invalid input to maximum
+ a = Commit.find_commit_range(users(:active), nil, nil, "$(uname>#{touchdir}/uh_oh)", nil)
+ assert !File.exists?("#{touchdir}/uh_oh"), "#{touchdir}/uh_oh should not exist, 'maximum' parameter of find_commit_range is exploitable"
+ assert_equal nil, a
+
+ # invalid input to minimum
+ a = Commit.find_commit_range(users(:active), nil, "31ce37fe365b3dc204300a3e4c396ad333ed0556 ; touch #{touchdir}/uh_oh", "31ce37fe365b3dc204300a3e4c396ad333ed0556", nil)
+ assert !File.exists?("#{touchdir}/uh_oh"), "#{touchdir}/uh_oh should not exist, 'minimum' parameter of find_commit_range is exploitable"
+ assert_equal nil, a
+
+ # invalid input to minimum
+ a = Commit.find_commit_range(users(:active), nil, "$(uname>#{touchdir}/uh_oh)", "31ce37fe365b3dc204300a3e4c396ad333ed0556", nil)
+ assert !File.exists?("#{touchdir}/uh_oh"), "#{touchdir}/uh_oh should not exist, 'minimum' parameter of find_commit_range is exploitable"
+ assert_equal nil, a
+
+ # invalid input to 'excludes'
+ a = Commit.find_commit_range(users(:active), nil, "31ce37fe365b3dc204300a3e4c396ad333ed0556", "077ba2ad3ea24a929091a9e6ce545c93199b8e57", ["4fe459abe02d9b365932b8f5dc419439ab4e2577 ; touch #{touchdir}/uh_oh"])
+ assert !File.exists?("#{touchdir}/uh_oh"), "#{touchdir}/uh_oh should not exist, 'excludes' parameter of find_commit_range is exploitable"
+ assert_equal nil, a
+
+ # invalid input to 'excludes'
+ a = Commit.find_commit_range(users(:active), nil, "31ce37fe365b3dc204300a3e4c396ad333ed0556", "077ba2ad3ea24a929091a9e6ce545c93199b8e57", ["$(uname>#{touchdir}/uh_oh)"])
+ assert !File.exists?("#{touchdir}/uh_oh"), "#{touchdir}/uh_oh should not exist, 'excludes' parameter of find_commit_range is exploitable"
+ assert_equal nil, a
+
+ end
+
+ end
+
+end
--- /dev/null
+require 'fileutils'
+require 'tmpdir'
+
+# Commit log for test.git.tar
+# master is the main branch
+# b1 is a branch off of master
+# tag1 is a tag
+#
+# 1de84a8 * b1
+# 077ba2a * master
+# 4fe459a * tag1
+# 31ce37f * foo
+
+module GitSetup
+ def setup
+ @tmpdir = Dir.mktmpdir()
+ #puts "setup #{@tmpdir}"
+ `cp test/test.git.tar #{@tmpdir} && cd #{@tmpdir} && tar xf test.git.tar`
+ Rails.configuration.git_repositories_dir = "#{@tmpdir}/test"
+ Commit.refresh_repositories
+ end
+
+ def teardown
+ #puts "teardown #{@tmpdir}"
+ FileUtils.remove_entry @tmpdir, true
+ end
+end
--- /dev/null
+require 'test_helper'
+load 'test/functional/arvados/v1/git_setup.rb'
+
+class Arvados::V1::JobReuseControllerTest < ActionController::TestCase
+ fixtures :repositories, :users, :jobs, :links
+
+ # See git_setup.rb for the commit log for test.git.tar
+ include GitSetup
+
+ test "test_reuse_job" do
+ @controller = Arvados::V1::JobsController.new
+ authorize_with :active
+ post :create, job: {
+ script: "hash",
+ script_version: "4fe459abe02d9b365932b8f5dc419439ab4e2577",
+ repository: "foo",
+ script_parameters: {
+ input: 'fa7aeb5140e2848d39b416daeef4ffc5+45',
+ an_integer: '1'
+ }
+ }
+ assert_response :success
+ assert_not_nil assigns(:object)
+ new_job = JSON.parse(@response.body)
+ assert_equal 'zzzzz-8i9sb-cjs4pklxxjykqqq', new_job['uuid']
+ assert_equal '4fe459abe02d9b365932b8f5dc419439ab4e2577', new_job['script_version']
+ end
+
+ test "test_reuse_job_range" do
+ @controller = Arvados::V1::JobsController.new
+ authorize_with :active
+ post :create, job: {
+ script: "hash",
+ minimum_script_version: "tag1",
+ script_version: "master",
+ repository: "foo",
+ script_parameters: {
+ input: 'fa7aeb5140e2848d39b416daeef4ffc5+45',
+ an_integer: '1'
+ }
+ }
+ assert_response :success
+ assert_not_nil assigns(:object)
+ new_job = JSON.parse(@response.body)
+ assert_equal 'zzzzz-8i9sb-cjs4pklxxjykqqq', new_job['uuid']
+ assert_equal '4fe459abe02d9b365932b8f5dc419439ab4e2577', new_job['script_version']
+ end
+
+ test "test_cannot_reuse_job_different_input" do
+ @controller = Arvados::V1::JobsController.new
+ authorize_with :active
+ post :create, job: {
+ script: "hash",
+ script_version: "4fe459abe02d9b365932b8f5dc419439ab4e2577",
+ repository: "foo",
+ script_parameters: {
+ input: 'fa7aeb5140e2848d39b416daeef4ffc5+45',
+ an_integer: '2'
+ }
+ }
+ assert_response :success
+ assert_not_nil assigns(:object)
+ new_job = JSON.parse(@response.body)
+ assert_not_equal 'zzzzz-8i9sb-cjs4pklxxjykqqq', new_job['uuid']
+ assert_equal '4fe459abe02d9b365932b8f5dc419439ab4e2577', new_job['script_version']
+ end
+
+ test "test_cannot_reuse_job_different_version" do
+ @controller = Arvados::V1::JobsController.new
+ authorize_with :active
+ post :create, job: {
+ script: "hash",
+ script_version: "master",
+ repository: "foo",
+ script_parameters: {
+ input: 'fa7aeb5140e2848d39b416daeef4ffc5+45',
+ an_integer: '2'
+ }
+ }
+ assert_response :success
+ assert_not_nil assigns(:object)
+ new_job = JSON.parse(@response.body)
+ assert_not_equal 'zzzzz-8i9sb-cjs4pklxxjykqqq', new_job['uuid']
+ assert_equal '077ba2ad3ea24a929091a9e6ce545c93199b8e57', new_job['script_version']
+ end
+
+ test "test_cannot_reuse_job_submitted_nondeterministic" do
+ @controller = Arvados::V1::JobsController.new
+ authorize_with :active
+ post :create, job: {
+ script: "hash",
+ script_version: "4fe459abe02d9b365932b8f5dc419439ab4e2577",
+ repository: "foo",
+ script_parameters: {
+ input: 'fa7aeb5140e2848d39b416daeef4ffc5+45',
+ an_integer: '1'
+ },
+ nondeterministic: true
+ }
+ assert_response :success
+ assert_not_nil assigns(:object)
+ new_job = JSON.parse(@response.body)
+ assert_not_equal 'zzzzz-8i9sb-cjs4pklxxjykqqq', new_job['uuid']
+ assert_equal '4fe459abe02d9b365932b8f5dc419439ab4e2577', new_job['script_version']
+ end
+
+ test "test_cannot_reuse_job_past_nondeterministic" do
+ @controller = Arvados::V1::JobsController.new
+ authorize_with :active
+ post :create, job: {
+ script: "hash2",
+ script_version: "4fe459abe02d9b365932b8f5dc419439ab4e2577",
+ repository: "foo",
+ script_parameters: {
+ input: 'fa7aeb5140e2848d39b416daeef4ffc5+45',
+ an_integer: '1'
+ }
+ }
+ assert_response :success
+ assert_not_nil assigns(:object)
+ new_job = JSON.parse(@response.body)
+ assert_not_equal 'zzzzz-8i9sb-cjs4pklxxjykyyy', new_job['uuid']
+ assert_equal '4fe459abe02d9b365932b8f5dc419439ab4e2577', new_job['script_version']
+ end
+
+ test "test_cannot_reuse_job_no_permission" do
+ @controller = Arvados::V1::JobsController.new
+ authorize_with :spectator
+ post :create, job: {
+ script: "hash",
+ script_version: "4fe459abe02d9b365932b8f5dc419439ab4e2577",
+ repository: "foo",
+ script_parameters: {
+ input: 'fa7aeb5140e2848d39b416daeef4ffc5+45',
+ an_integer: '1'
+ }
+ }
+ assert_response :success
+ assert_not_nil assigns(:object)
+ new_job = JSON.parse(@response.body)
+ assert_not_equal 'zzzzz-8i9sb-cjs4pklxxjykqqq', new_job['uuid']
+ assert_equal '4fe459abe02d9b365932b8f5dc419439ab4e2577', new_job['script_version']
+ end
+
+ test "test_cannot_reuse_job_excluded" do
+ @controller = Arvados::V1::JobsController.new
+ authorize_with :active
+ post :create, job: {
+ script: "hash",
+ minimum_script_version: "31ce37fe365b3dc204300a3e4c396ad333ed0556",
+ script_version: "master",
+ repository: "foo",
+ exclude_script_versions: ["tag1"],
+ script_parameters: {
+ input: 'fa7aeb5140e2848d39b416daeef4ffc5+45',
+ an_integer: '1'
+ }
+ }
+ assert_response :success
+ assert_not_nil assigns(:object)
+ new_job = JSON.parse(@response.body)
+ assert_not_equal 'zzzzz-8i9sb-cjs4pklxxjykqqq', new_job['uuid']
+ assert_equal '077ba2ad3ea24a929091a9e6ce545c93199b8e57', new_job['script_version']
+ end
+
+
+end
require 'test_helper'
+load 'test/functional/arvados/v1/git_setup.rb'
class Arvados::V1::JobsControllerTest < ActionController::TestCase
+ include GitSetup
+
test "submit a job" do
authorize_with :active
post :create, job: {
script: "hash",
script_version: "master",
+ repository: "foo",
script_parameters: {}
}
assert_response :success
new_job = JSON.parse(@response.body)
assert_not_nil new_job['uuid']
assert_not_nil new_job['script_version'].match(/^[0-9a-f]{40}$/)
+ # Default: not persistent
+ assert_equal false, new_job['output_is_persistent']
end
test "normalize output and log uuids when creating job" do
script: "hash",
script_version: "master",
script_parameters: {},
+ repository: "foo",
started_at: Time.now,
finished_at: Time.now,
running: false,
}
assert_response :success
end
+
+ test "search jobs by uuid with >= query" do
+ authorize_with :active
+ get :index, {
+ filters: [['uuid', '>=', 'zzzzz-8i9sb-pshmckwoma9plh7']]
+ }
+ assert_response :success
+ found = assigns(:objects).collect(&:uuid)
+ assert_equal true, !!found.index('zzzzz-8i9sb-pshmckwoma9plh7')
+ assert_equal false, !!found.index('zzzzz-8i9sb-4cf0nhn6xte809j')
+ end
+
+ test "search jobs by uuid with <= query" do
+ authorize_with :active
+ get :index, {
+ filters: [['uuid', '<=', 'zzzzz-8i9sb-pshmckwoma9plh7']]
+ }
+ assert_response :success
+ found = assigns(:objects).collect(&:uuid)
+ assert_equal true, !!found.index('zzzzz-8i9sb-pshmckwoma9plh7')
+ assert_equal true, !!found.index('zzzzz-8i9sb-4cf0nhn6xte809j')
+ end
+
+ test "search jobs by uuid with >= and <= query" do
+ authorize_with :active
+ get :index, {
+ filters: [['uuid', '>=', 'zzzzz-8i9sb-pshmckwoma9plh7'],
+ ['uuid', '<=', 'zzzzz-8i9sb-pshmckwoma9plh7']]
+ }
+ assert_response :success
+ found = assigns(:objects).collect(&:uuid)
+ assert_equal found, ['zzzzz-8i9sb-pshmckwoma9plh7']
+ end
+
+ test "search jobs by uuid with < query" do
+ authorize_with :active
+ get :index, {
+ filters: [['uuid', '<', 'zzzzz-8i9sb-pshmckwoma9plh7']]
+ }
+ assert_response :success
+ found = assigns(:objects).collect(&:uuid)
+ assert_equal false, !!found.index('zzzzz-8i9sb-pshmckwoma9plh7')
+ assert_equal true, !!found.index('zzzzz-8i9sb-4cf0nhn6xte809j')
+ end
+
+ test "search jobs by uuid with like query" do
+ authorize_with :active
+ get :index, {
+ filters: [['uuid', 'like', '%hmckwoma9pl%']]
+ }
+ assert_response :success
+ found = assigns(:objects).collect(&:uuid)
+ assert_equal found, ['zzzzz-8i9sb-pshmckwoma9plh7']
+ end
+
+ test "search jobs by uuid with 'in' query" do
+ authorize_with :active
+ get :index, {
+ filters: [['uuid', 'in', ['zzzzz-8i9sb-4cf0nhn6xte809j',
+ 'zzzzz-8i9sb-pshmckwoma9plh7']]]
+ }
+ assert_response :success
+ found = assigns(:objects).collect(&:uuid)
+ assert_equal found.sort, ['zzzzz-8i9sb-4cf0nhn6xte809j',
+ 'zzzzz-8i9sb-pshmckwoma9plh7']
+ end
+
+ test "search jobs by started_at with < query" do
+ authorize_with :active
+ get :index, {
+ filters: [['started_at', '<', Time.now.to_s]]
+ }
+ assert_response :success
+ found = assigns(:objects).collect(&:uuid)
+ assert_equal true, !!found.index('zzzzz-8i9sb-pshmckwoma9plh7')
+ end
+
+ test "search jobs by started_at with > query" do
+ authorize_with :active
+ get :index, {
+ filters: [['started_at', '>', Time.now.to_s]]
+ }
+ assert_response :success
+ assert_equal 0, assigns(:objects).count
+ end
+
+ test "search jobs by started_at with >= query on metric date" do
+ authorize_with :active
+ get :index, {
+ filters: [['started_at', '>=', '2014-01-01']]
+ }
+ assert_response :success
+ found = assigns(:objects).collect(&:uuid)
+ assert_equal true, !!found.index('zzzzz-8i9sb-pshmckwoma9plh7')
+ end
+
+ test "search jobs by started_at with >= query on metric date and time" do
+ authorize_with :active
+ get :index, {
+ filters: [['started_at', '>=', '2014-01-01 01:23:45']]
+ }
+ assert_response :success
+ found = assigns(:objects).collect(&:uuid)
+ assert_equal true, !!found.index('zzzzz-8i9sb-pshmckwoma9plh7')
+ end
+
+ test "search jobs with 'any' operator" do
+ authorize_with :active
+ get :index, {
+ where: { any: ['contains', 'pshmckw'] }
+ }
+ assert_response :success
+ found = assigns(:objects).collect(&:uuid)
+ assert_equal 0, found.index('zzzzz-8i9sb-pshmckwoma9plh7')
+ assert_equal 1, found.count
+ end
+
+ test "search jobs by nonexistent column with < query" do
+ authorize_with :active
+ get :index, {
+ filters: [['is_borked', '<', 'fizzbuzz']]
+ }
+ assert_response 422
+ end
+
+
end
end
end
+ test "search keep_disks by service_port with >= query" do
+ authorize_with :active
+ get :index, {
+ filters: [['service_port', '>=', 25107]]
+ }
+ assert_response :success
+ assert_equal true, assigns(:objects).any?
+ end
+
+ test "search keep_disks by service_port with < query" do
+ authorize_with :active
+ get :index, {
+ filters: [['service_port', '<', 25107]]
+ }
+ assert_response :success
+ assert_equal false, assigns(:objects).any?
+ end
+
+ test "search keep_disks with 'any' operator" do
+ authorize_with :active
+ get :index, {
+ where: { any: ['contains', 'o2t1q5w'] }
+ }
+ assert_response :success
+ found = assigns(:objects).collect(&:uuid)
+ assert_equal true, !!found.index('zzzzz-penuu-5w2o2t1q5wy7fhn')
+ end
+
end
properties: {username: 'testusername'},
link_class: 'test',
name: 'encoding',
- tail_kind: 'arvados#user',
tail_uuid: users(:admin).uuid,
- head_kind: 'arvados#virtualMachine',
head_uuid: virtual_machines(:testvm).uuid
}
authorize_with :admin
assert_equal false, assigns(:object).properties.has_key?(:username)
end
end
-
+
+ test "head must exist" do
+ link = {
+ link_class: 'test',
+ name: 'stuff',
+ tail_uuid: users(:active).uuid,
+ head_uuid: 'zzzzz-tpzed-xyzxyzxerrrorxx'
+ }
+ authorize_with :admin
+ post :create, link: link
+ assert_response 422
+ end
+
+ test "tail must exist" do
+ link = {
+ link_class: 'test',
+ name: 'stuff',
+ head_uuid: users(:active).uuid,
+ tail_uuid: 'zzzzz-tpzed-xyzxyzxerrrorxx'
+ }
+ authorize_with :admin
+ post :create, link: link
+ assert_response 422
+ end
+
+ test "head and tail exist, head_kind and tail_kind are returned" do
+ link = {
+ link_class: 'test',
+ name: 'stuff',
+ head_uuid: users(:active).uuid,
+ tail_uuid: users(:spectator).uuid,
+ }
+ authorize_with :admin
+ post :create, link: link
+ assert_response :success
+ l = JSON.parse(@response.body)
+ assert 'arvados#user', l['head_kind']
+ assert 'arvados#user', l['tail_kind']
+ end
+
+ test "can supply head_kind and tail_kind without error" do
+ link = {
+ link_class: 'test',
+ name: 'stuff',
+ head_uuid: users(:active).uuid,
+ tail_uuid: users(:spectator).uuid,
+ head_kind: "arvados#user",
+ tail_kind: "arvados#user",
+ }
+ authorize_with :admin
+ post :create, link: link
+ assert_response :success
+ l = JSON.parse(@response.body)
+ assert 'arvados#user', l['head_kind']
+ assert 'arvados#user', l['tail_kind']
+ end
+
+ test "tail must be visible by user" do
+ link = {
+ link_class: 'test',
+ name: 'stuff',
+ head_uuid: users(:active).uuid,
+ tail_uuid: virtual_machines(:testvm).uuid
+ }
+ authorize_with :active
+ post :create, link: link
+ assert_response 422
+ end
+
+ test "filter links with 'is_a' operator" do
+ authorize_with :admin
+ get :index, {
+ filters: [ ['tail_uuid', 'is_a', 'arvados#user'] ]
+ }
+ assert_response :success
+ found = assigns(:objects)
+ assert_not_equal 0, found.count
+ assert_equal found.count, (found.select { |f| f.tail_uuid.match /[a-z0-9]{5}-tpzed-[a-z0-9]{15}/}).count
+ end
+
+ test "filter links with 'is_a' operator with more than one" do
+ authorize_with :admin
+ get :index, {
+ filters: [ ['tail_uuid', 'is_a', ['arvados#user', 'arvados#group'] ] ],
+ }
+ assert_response :success
+ found = assigns(:objects)
+ assert_not_equal 0, found.count
+ assert_equal found.count, (found.select { |f| f.tail_uuid.match /[a-z0-9]{5}-(tpzed|j7d0g)-[a-z0-9]{15}/}).count
+ end
+
+ test "filter links with 'is_a' operator with bogus type" do
+ authorize_with :admin
+ get :index, {
+ filters: [ ['tail_uuid', 'is_a', ['arvados#bogus'] ] ],
+ }
+ assert_response :success
+ found = assigns(:objects)
+ assert_equal 0, found.count
+ end
+
+ test "filter links with 'is_a' operator with collection" do
+ authorize_with :admin
+ get :index, {
+ filters: [ ['head_uuid', 'is_a', ['arvados#collection'] ] ],
+ }
+ assert_response :success
+ found = assigns(:objects)
+ assert_not_equal 0, found.count
+ assert_equal found.count, (found.select { |f| f.head_uuid.match /[a-f0-9]{32}\+\d+/}).count
+ end
+
+ test "test can still use where tail_kind" do
+ authorize_with :admin
+ get :index, {
+ where: { tail_kind: 'arvados#user' }
+ }
+ assert_response :success
+ found = assigns(:objects)
+ assert_not_equal 0, found.count
+ assert_equal found.count, (found.select { |f| f.tail_uuid.match /[a-z0-9]{5}-tpzed-[a-z0-9]{15}/}).count
+ end
+
+ test "test can still use where head_kind" do
+ authorize_with :admin
+ get :index, {
+ where: { head_kind: 'arvados#user' }
+ }
+ assert_response :success
+ found = assigns(:objects)
+ assert_not_equal 0, found.count
+ assert_equal found.count, (found.select { |f| f.head_uuid.match /[a-z0-9]{5}-tpzed-[a-z0-9]{15}/}).count
+ end
+
+ test "test can still use filter tail_kind" do
+ authorize_with :admin
+ get :index, {
+ filters: [ ['tail_kind', '=', 'arvados#user'] ]
+ }
+ assert_response :success
+ found = assigns(:objects)
+ assert_not_equal 0, found.count
+ assert_equal found.count, (found.select { |f| f.tail_uuid.match /[a-z0-9]{5}-tpzed-[a-z0-9]{15}/}).count
+ end
+
+ test "test can still use filter head_kind" do
+ authorize_with :admin
+ get :index, {
+ filters: [ ['head_kind', '=', 'arvados#user'] ]
+ }
+ assert_response :success
+ found = assigns(:objects)
+ assert_not_equal 0, found.count
+ assert_equal found.count, (found.select { |f| f.head_uuid.match /[a-z0-9]{5}-tpzed-[a-z0-9]{15}/}).count
+ end
+
+ test "head_kind matches head_uuid" do
+ link = {
+ link_class: 'test',
+ name: 'stuff',
+ head_uuid: groups(:public).uuid,
+ head_kind: "arvados#user",
+ tail_uuid: users(:spectator).uuid,
+ tail_kind: "arvados#user",
+ }
+ authorize_with :admin
+ post :create, link: link
+ assert_response 422
+ end
+
+ test "tail_kind matches tail_uuid" do
+ link = {
+ link_class: 'test',
+ name: 'stuff',
+ head_uuid: users(:active).uuid,
+ head_kind: "arvados#user",
+ tail_uuid: groups(:public).uuid,
+ tail_kind: "arvados#user",
+ }
+ authorize_with :admin
+ post :create, link: link
+ assert_response 422
+ end
+
end
require 'test_helper'
class Arvados::V1::LogsControllerTest < ActionController::TestCase
+ fixtures :logs
+
+ test "non-admins can read their own logs" do
+ authorize_with :active
+ post :create, log: {summary: "test log"}
+ assert_response :success
+ uuid = JSON.parse(@response.body)['uuid']
+ assert_not_nil uuid
+ get :show, {id: uuid}
+ assert_response(:success, "failed to load created log")
+ assert_equal("test log", assigns(:object).summary,
+ "loaded wrong log after creation")
+ end
+
+ test "test can still use where object_kind" do
+ authorize_with :admin
+ get :index, {
+ where: { object_kind: 'arvados#user' }
+ }
+ assert_response :success
+ found = assigns(:objects)
+ assert_not_equal 0, found.count
+ assert_equal found.count, (found.select { |f| f.object_uuid.match /[a-z0-9]{5}-tpzed-[a-z0-9]{15}/}).count
+ l = JSON.parse(@response.body)
+ assert_equal 'arvados#user', l['items'][0]['object_kind']
+ end
+
+ test "test can still use filter object_kind" do
+ authorize_with :admin
+ get :index, {
+ filters: [ ['object_kind', '=', 'arvados#user'] ]
+ }
+ assert_response :success
+ found = assigns(:objects)
+ assert_not_equal 0, found.count
+ assert_equal found.count, (found.select { |f| f.object_uuid.match /[a-z0-9]{5}-tpzed-[a-z0-9]{15}/}).count
+ end
+
end
test "should get fresh discovery document" do
MAX_SCHEMA_AGE = 60
- get :discovery_rest_description
+ get :index
assert_response :success
discovery_doc = JSON.parse(@response.body)
assert_equal 'discovery#restDescription', discovery_doc['kind']
require 'test_helper'
class Arvados::V1::UsersControllerTest < ActionController::TestCase
+ include CurrentApiClient
+
+ setup do
+ @all_links_at_start = Link.all
+ @vm_uuid = virtual_machines(:testvm).uuid
+ end
test "activate a user after signing UA" do
authorize_with :inactive_but_signed_user_agreement
assert_equal true, me['is_active']
end
+ test "respond 401 if given token exists but user record is missing" do
+ authorize_with :valid_token_deleted_user
+ get :current, {format: :json}
+ assert_response 401
+ end
+
+ test "create new user with user as input" do
+ authorize_with :admin
+ post :create, user: {
+ first_name: "test_first_name",
+ last_name: "test_last_name",
+ email: "foo@example.com"
+ }
+ assert_response :success
+ created = JSON.parse(@response.body)
+ assert_equal 'test_first_name', created['first_name']
+ assert_not_nil created['uuid'], 'expected uuid for the newly created user'
+ assert_not_nil created['email'], 'expected non-nil email'
+ assert_nil created['identity_url'], 'expected no identity_url'
+ end
+
+ test "create user with user, vm and repo as input" do
+ authorize_with :admin
+ repo_name = 'test_repo'
+
+ post :setup, {
+ repo_name: repo_name,
+ openid_prefix: 'https://www.google.com/accounts/o8/id',
+ user: {
+ uuid: 'zzzzz-tpzed-abcdefghijklmno',
+ first_name: "in_create_test_first_name",
+ last_name: "test_last_name",
+ email: "foo@example.com"
+ }
+ }
+ assert_response :success
+ response_items = JSON.parse(@response.body)['items']
+
+ created = find_obj_in_resp response_items, 'User', nil
+
+ assert_equal 'in_create_test_first_name', created['first_name']
+ assert_not_nil created['uuid'], 'expected non-null uuid for the new user'
+ assert_equal 'zzzzz-tpzed-abcdefghijklmno', created['uuid']
+ assert_not_nil created['email'], 'expected non-nil email'
+ assert_nil created['identity_url'], 'expected no identity_url'
+
+ # arvados#user, repo link and link add user to 'All users' group
+ verify_num_links @all_links_at_start, 4
+
+ verify_link response_items, 'arvados#user', true, 'permission', 'can_login',
+ created['uuid'], created['email'], 'arvados#user', false, 'User'
+
+ verify_link response_items, 'arvados#repository', true, 'permission', 'can_write',
+ repo_name, created['uuid'], 'arvados#repository', true, 'Repository'
+
+ verify_link response_items, 'arvados#group', true, 'permission', 'can_read',
+ 'All users', created['uuid'], 'arvados#group', true, 'Group'
+
+ verify_link response_items, 'arvados#virtualMachine', false, 'permission', 'can_login',
+ nil, created['uuid'], 'arvados#virtualMachine', false, 'VirtualMachine'
+
+ verify_system_group_permission_link_for created['uuid']
+
+ # invoke setup again with the same data
+ post :setup, {
+ repo_name: repo_name,
+ vm_uuid: @vm_uuid,
+ openid_prefix: 'https://www.google.com/accounts/o8/id',
+ user: {
+ uuid: 'zzzzz-tpzed-abcdefghijklmno',
+ first_name: "in_create_test_first_name",
+ last_name: "test_last_name",
+ email: "foo@example.com"
+ }
+ }
+
+ response_items = JSON.parse(@response.body)['items']
+
+ created = find_obj_in_resp response_items, 'User', nil
+ assert_equal 'in_create_test_first_name', created['first_name']
+ assert_not_nil created['uuid'], 'expected non-null uuid for the new user'
+ assert_equal 'zzzzz-tpzed-abcdefghijklmno', created['uuid']
+ assert_not_nil created['email'], 'expected non-nil email'
+ assert_nil created['identity_url'], 'expected no identity_url'
+
+ # arvados#user, repo link and link add user to 'All users' group
+ verify_num_links @all_links_at_start, 5
+
+ verify_link response_items, 'arvados#repository', true, 'permission', 'can_write',
+ repo_name, created['uuid'], 'arvados#repository', true, 'Repository'
+
+ verify_link response_items, 'arvados#group', true, 'permission', 'can_read',
+ 'All users', created['uuid'], 'arvados#group', true, 'Group'
+
+ verify_link response_items, 'arvados#virtualMachine', true, 'permission', 'can_login',
+ @vm_uuid, created['uuid'], 'arvados#virtualMachine', false, 'VirtualMachine'
+
+ verify_system_group_permission_link_for created['uuid']
+ end
+
+ test "setup user with bogus uuid and expect error" do
+ authorize_with :admin
+
+ post :setup, {
+ uuid: 'bogus_uuid',
+ repo_name: 'test_repo',
+ vm_uuid: @vm_uuid
+ }
+ response_body = JSON.parse(@response.body)
+ response_errors = response_body['errors']
+ assert_not_nil response_errors, 'Expected error in response'
+ assert (response_errors.first.include? 'Path not found'), 'Expected 404'
+ end
+
+ test "setup user with bogus uuid in user and expect error" do
+ authorize_with :admin
+
+ post :setup, {
+ user: {uuid: 'bogus_uuid'},
+ repo_name: 'test_repo',
+ vm_uuid: @vm_uuid,
+ openid_prefix: 'https://www.google.com/accounts/o8/id'
+ }
+ response_body = JSON.parse(@response.body)
+ response_errors = response_body['errors']
+ assert_not_nil response_errors, 'Expected error in response'
+ assert (response_errors.first.include? 'ArgumentError: Require user email'),
+ 'Expected RuntimeError'
+ end
+
+ test "setup user with no uuid and user, expect error" do
+ authorize_with :admin
+
+ post :setup, {
+ repo_name: 'test_repo',
+ vm_uuid: @vm_uuid,
+ openid_prefix: 'https://www.google.com/accounts/o8/id'
+ }
+ response_body = JSON.parse(@response.body)
+ response_errors = response_body['errors']
+ assert_not_nil response_errors, 'Expected error in response'
+ assert (response_errors.first.include? 'Required uuid or user'),
+ 'Expected ArgumentError'
+ end
+
+ test "setup user with no uuid and email, expect error" do
+ authorize_with :admin
+
+ post :setup, {
+ user: {},
+ repo_name: 'test_repo',
+ vm_uuid: @vm_uuid,
+ openid_prefix: 'https://www.google.com/accounts/o8/id'
+ }
+ response_body = JSON.parse(@response.body)
+ response_errors = response_body['errors']
+ assert_not_nil response_errors, 'Expected error in response'
+ assert (response_errors.first.include? '<ArgumentError: Require user email'),
+ 'Expected ArgumentError'
+ end
+
+ test "invoke setup with existing uuid, vm and repo and verify links" do
+ authorize_with :inactive
+ get :current
+ assert_response :success
+ inactive_user = JSON.parse(@response.body)
+
+ authorize_with :admin
+
+ post :setup, {
+ uuid: inactive_user['uuid'],
+ repo_name: 'test_repo',
+ vm_uuid: @vm_uuid
+ }
+
+ assert_response :success
+
+ response_items = JSON.parse(@response.body)['items']
+ resp_obj = find_obj_in_resp response_items, 'User', nil
+
+ assert_not_nil resp_obj['uuid'], 'expected uuid for the new user'
+ assert_equal inactive_user['uuid'], resp_obj['uuid']
+ assert_equal inactive_user['email'], resp_obj['email'],
+ 'expecting inactive user email'
+
+ # expect repo and vm links
+ verify_link response_items, 'arvados#repository', true, 'permission', 'can_write',
+ 'test_repo', resp_obj['uuid'], 'arvados#repository', true, 'Repository'
+
+ verify_link response_items, 'arvados#virtualMachine', true, 'permission', 'can_login',
+ @vm_uuid, resp_obj['uuid'], 'arvados#virtualMachine', false, 'VirtualMachine'
+ end
+
+ test "invoke setup with existing uuid in user, verify response" do
+ authorize_with :inactive
+ get :current
+ assert_response :success
+ inactive_user = JSON.parse(@response.body)
+
+ authorize_with :admin
+
+ post :setup, {
+ user: {uuid: inactive_user['uuid']},
+ openid_prefix: 'https://www.google.com/accounts/o8/id'
+ }
+
+ assert_response :success
+
+ response_items = JSON.parse(@response.body)['items']
+ resp_obj = find_obj_in_resp response_items, 'User', nil
+
+ assert_not_nil resp_obj['uuid'], 'expected uuid for the new user'
+ assert_equal inactive_user['uuid'], resp_obj['uuid']
+ assert_equal inactive_user['email'], resp_obj['email'],
+ 'expecting inactive user email'
+ end
+
+ test "invoke setup with existing uuid but different email, expect original email" do
+ authorize_with :inactive
+ get :current
+ assert_response :success
+ inactive_user = JSON.parse(@response.body)
+
+ authorize_with :admin
+
+ post :setup, {
+ uuid: inactive_user['uuid'],
+ user: {email: 'junk_email'}
+ }
+
+ assert_response :success
+
+ response_items = JSON.parse(@response.body)['items']
+ resp_obj = find_obj_in_resp response_items, 'User', nil
+
+ assert_not_nil resp_obj['uuid'], 'expected uuid for the new user'
+ assert_equal inactive_user['uuid'], resp_obj['uuid']
+ assert_equal inactive_user['email'], resp_obj['email'],
+ 'expecting inactive user email'
+ end
+
+ test "setup user with valid email and repo as input" do
+ authorize_with :admin
+
+ post :setup, {
+ repo_name: 'test_repo',
+ user: {email: 'foo@example.com'},
+ openid_prefix: 'https://www.google.com/accounts/o8/id'
+ }
+
+ assert_response :success
+ response_items = JSON.parse(@response.body)['items']
+ response_object = find_obj_in_resp response_items, 'User', nil
+ assert_not_nil response_object['uuid'], 'expected uuid for the new user'
+ assert_equal response_object['email'], 'foo@example.com', 'expected given email'
+
+ # four extra links; system_group, login, group and repo perms
+ verify_num_links @all_links_at_start, 4
+ end
+
+ test "setup user with fake vm and expect error" do
+ authorize_with :admin
+
+ post :setup, {
+ repo_name: 'test_repo',
+ vm_uuid: 'no_such_vm',
+ user: {email: 'foo@example.com'},
+ openid_prefix: 'https://www.google.com/accounts/o8/id'
+ }
+
+ response_body = JSON.parse(@response.body)
+ response_errors = response_body['errors']
+ assert_not_nil response_errors, 'Expected error in response'
+ assert (response_errors.first.include? "No vm found for no_such_vm"),
+ 'Expected RuntimeError: No vm found for no_such_vm'
+ end
+
+ test "setup user with valid email, repo and real vm as input" do
+ authorize_with :admin
+
+ post :setup, {
+ repo_name: 'test_repo',
+ openid_prefix: 'https://www.google.com/accounts/o8/id',
+ vm_uuid: @vm_uuid,
+ user: {email: 'foo@example.com'}
+ }
+
+ assert_response :success
+ response_items = JSON.parse(@response.body)['items']
+ response_object = find_obj_in_resp response_items, 'User', nil
+ assert_not_nil response_object['uuid'], 'expected uuid for the new user'
+ assert_equal response_object['email'], 'foo@example.com', 'expected given email'
+
+ # five extra links; system_group, login, group, vm, repo
+ verify_num_links @all_links_at_start, 5
+ end
+
+ test "setup user with valid email, no vm and repo as input" do
+ authorize_with :admin
+
+ post :setup, {
+ user: {email: 'foo@example.com'},
+ openid_prefix: 'https://www.google.com/accounts/o8/id'
+ }
+
+ assert_response :success
+ response_items = JSON.parse(@response.body)['items']
+ response_object = find_obj_in_resp response_items, 'User', nil
+ assert_not_nil response_object['uuid'], 'expected uuid for new user'
+ assert_equal response_object['email'], 'foo@example.com', 'expected given email'
+
+ # three extra links; system_group, login, and group
+ verify_num_links @all_links_at_start, 3
+ end
+
+ test "setup user with email, first name, repo name and vm uuid" do
+ authorize_with :admin
+
+ post :setup, {
+ openid_prefix: 'https://www.google.com/accounts/o8/id',
+ repo_name: 'test_repo',
+ vm_uuid: @vm_uuid,
+ user: {
+ first_name: 'test_first_name',
+ email: 'foo@example.com'
+ }
+ }
+
+ assert_response :success
+ response_items = JSON.parse(@response.body)['items']
+ response_object = find_obj_in_resp response_items, 'User', nil
+ assert_not_nil response_object['uuid'], 'expected uuid for new user'
+ assert_equal response_object['email'], 'foo@example.com', 'expected given email'
+ assert_equal 'test_first_name', response_object['first_name'],
+ 'expecting first name'
+
+ # five extra links; system_group, login, group, repo and vm
+ verify_num_links @all_links_at_start, 5
+ end
+
+ test "setup user twice with email and check two different objects created" do
+ authorize_with :admin
+
+ post :setup, {
+ openid_prefix: 'https://www.google.com/accounts/o8/id',
+ repo_name: 'test_repo',
+ user: {
+ email: 'foo@example.com'
+ }
+ }
+
+ assert_response :success
+ response_items = JSON.parse(@response.body)['items']
+ response_object = find_obj_in_resp response_items, 'User', nil
+ assert_not_nil response_object['uuid'], 'expected uuid for new user'
+ assert_equal response_object['email'], 'foo@example.com', 'expected given email'
+ # system_group, openid, group, and repo. No vm link.
+ verify_num_links @all_links_at_start, 4
+
+ # create again
+ post :setup, {
+ user: {email: 'foo@example.com'},
+ openid_prefix: 'https://www.google.com/accounts/o8/id'
+ }
+
+ assert_response :success
+ response_items = JSON.parse(@response.body)['items']
+ response_object2 = find_obj_in_resp response_items, 'User', nil
+ assert_not_equal response_object['uuid'], response_object2['uuid'],
+ 'expected same uuid as first create operation'
+ assert_equal response_object['email'], 'foo@example.com', 'expected given email'
+
+ # +1 extra login link +1 extra system_group link pointing to the new User
+ verify_num_links @all_links_at_start, 6
+ end
+
+ test "setup user with openid prefix" do
+ authorize_with :admin
+
+ post :setup, {
+ repo_name: 'test_repo',
+ openid_prefix: 'http://www.example.com/account',
+ user: {
+ first_name: "in_create_test_first_name",
+ last_name: "test_last_name",
+ email: "foo@example.com"
+ }
+ }
+
+ assert_response :success
+
+ response_items = JSON.parse(@response.body)['items']
+ created = find_obj_in_resp response_items, 'User', nil
+
+ assert_equal 'in_create_test_first_name', created['first_name']
+ assert_not_nil created['uuid'], 'expected uuid for new user'
+ assert_not_nil created['email'], 'expected non-nil email'
+ assert_nil created['identity_url'], 'expected no identity_url'
+
+ # verify links
+ # four new links: system_group, arvados#user, repo, and 'All users' group.
+ verify_num_links @all_links_at_start, 4
+
+ verify_link response_items, 'arvados#user', true, 'permission', 'can_login',
+ created['uuid'], created['email'], 'arvados#user', false, 'User'
+
+ verify_link response_items, 'arvados#repository', true, 'permission', 'can_write',
+ 'test_repo', created['uuid'], 'arvados#repository', true, 'Repository'
+
+ verify_link response_items, 'arvados#group', true, 'permission', 'can_read',
+ 'All users', created['uuid'], 'arvados#group', true, 'Group'
+
+ verify_link response_items, 'arvados#virtualMachine', false, 'permission', 'can_login',
+ nil, created['uuid'], 'arvados#virtualMachine', false, 'VirtualMachine'
+ end
+
+ test "invoke setup with no openid prefix, expect error" do
+ authorize_with :admin
+
+ post :setup, {
+ repo_name: 'test_repo',
+ user: {
+ first_name: "in_create_test_first_name",
+ last_name: "test_last_name",
+ email: "foo@example.com"
+ }
+ }
+
+ response_body = JSON.parse(@response.body)
+ response_errors = response_body['errors']
+ assert_not_nil response_errors, 'Expected error in response'
+ assert (response_errors.first.include? 'openid_prefix parameter is missing'),
+ 'Expected ArgumentError'
+ end
+
+ test "setup user with user, vm and repo and verify links" do
+ authorize_with :admin
+
+ post :setup, {
+ user: {
+ first_name: "in_create_test_first_name",
+ last_name: "test_last_name",
+ email: "foo@example.com"
+ },
+ vm_uuid: @vm_uuid,
+ repo_name: 'test_repo',
+ openid_prefix: 'https://www.google.com/accounts/o8/id'
+ }
+
+ assert_response :success
+
+ response_items = JSON.parse(@response.body)['items']
+ created = find_obj_in_resp response_items, 'User', nil
+
+ assert_equal 'in_create_test_first_name', created['first_name']
+ assert_not_nil created['uuid'], 'expected uuid for new user'
+ assert_not_nil created['email'], 'expected non-nil email'
+ assert_nil created['identity_url'], 'expected no identity_url'
+
+ # five new links: system_group, arvados#user, repo, vm and 'All
+ # users' group link
+ verify_num_links @all_links_at_start, 5
+
+ verify_link response_items, 'arvados#user', true, 'permission', 'can_login',
+ created['uuid'], created['email'], 'arvados#user', false, 'User'
+
+ verify_link response_items, 'arvados#repository', true, 'permission', 'can_write',
+ 'test_repo', created['uuid'], 'arvados#repository', true, 'Repository'
+
+ verify_link response_items, 'arvados#group', true, 'permission', 'can_read',
+ 'All users', created['uuid'], 'arvados#group', true, 'Group'
+
+ verify_link response_items, 'arvados#virtualMachine', true, 'permission', 'can_login',
+ @vm_uuid, created['uuid'], 'arvados#virtualMachine', false, 'VirtualMachine'
+ end
+
+ test "create user as non admin user and expect error" do
+ authorize_with :active
+
+ post :create, {
+ user: {email: 'foo@example.com'}
+ }
+
+ response_body = JSON.parse(@response.body)
+ response_errors = response_body['errors']
+ assert_not_nil response_errors, 'Expected error in response'
+ assert (response_errors.first.include? 'PermissionDenied'),
+ 'Expected PermissionDeniedError'
+ end
+
+ test "setup user as non admin user and expect error" do
+ authorize_with :active
+
+ post :setup, {
+ openid_prefix: 'https://www.google.com/accounts/o8/id',
+ user: {email: 'foo@example.com'}
+ }
+
+ response_body = JSON.parse(@response.body)
+ response_errors = response_body['errors']
+ assert_not_nil response_errors, 'Expected error in response'
+ assert (response_errors.first.include? 'Forbidden'),
+ 'Expected Forbidden error'
+ end
+
+ test "setup user in multiple steps and verify response" do
+ authorize_with :admin
+
+ post :setup, {
+ openid_prefix: 'http://www.example.com/account',
+ user: {
+ email: "foo@example.com"
+ }
+ }
+
+ assert_response :success
+ response_items = JSON.parse(@response.body)['items']
+ created = find_obj_in_resp response_items, 'User', nil
+
+ assert_not_nil created['uuid'], 'expected uuid for new user'
+ assert_not_nil created['email'], 'expected non-nil email'
+ assert_equal created['email'], 'foo@example.com', 'expected input email'
+
+ # three new links: system_group, arvados#user, and 'All users' group.
+ verify_num_links @all_links_at_start, 3
+
+ verify_link response_items, 'arvados#user', true, 'permission', 'can_login',
+ created['uuid'], created['email'], 'arvados#user', false, 'User'
+
+ verify_link response_items, 'arvados#group', true, 'permission', 'can_read',
+ 'All users', created['uuid'], 'arvados#group', true, 'Group'
+
+ verify_link response_items, 'arvados#repository', false, 'permission', 'can_write',
+ 'test_repo', created['uuid'], 'arvados#repository', true, 'Repository'
+
+ verify_link response_items, 'arvados#virtualMachine', false, 'permission', 'can_login',
+ nil, created['uuid'], 'arvados#virtualMachine', false, 'VirtualMachine'
+
+ # invoke setup with a repository
+ post :setup, {
+ openid_prefix: 'http://www.example.com/account',
+ repo_name: 'new_repo',
+ uuid: created['uuid']
+ }
+
+ assert_response :success
+
+ response_items = JSON.parse(@response.body)['items']
+ created = find_obj_in_resp response_items, 'User', nil
+
+ assert_equal 'foo@example.com', created['email'], 'expected input email'
+
+ # verify links
+ verify_link response_items, 'arvados#group', true, 'permission', 'can_read',
+ 'All users', created['uuid'], 'arvados#group', true, 'Group'
+
+ verify_link response_items, 'arvados#repository', true, 'permission', 'can_write',
+ 'new_repo', created['uuid'], 'arvados#repository', true, 'Repository'
+
+ verify_link response_items, 'arvados#virtualMachine', false, 'permission', 'can_login',
+ nil, created['uuid'], 'arvados#virtualMachine', false, 'VirtualMachine'
+
+ # invoke setup with a vm_uuid
+ post :setup, {
+ vm_uuid: @vm_uuid,
+ openid_prefix: 'http://www.example.com/account',
+ user: {
+ email: 'junk_email'
+ },
+ uuid: created['uuid']
+ }
+
+ assert_response :success
+
+ response_items = JSON.parse(@response.body)['items']
+ created = find_obj_in_resp response_items, 'User', nil
+
+ assert_equal created['email'], 'foo@example.com', 'expected original email'
+
+ # verify links
+ verify_link response_items, 'arvados#group', true, 'permission', 'can_read',
+ 'All users', created['uuid'], 'arvados#group', true, 'Group'
+
+ # since no repo name in input, we won't get any; even though user has one
+ verify_link response_items, 'arvados#repository', false, 'permission', 'can_write',
+ 'new_repo', created['uuid'], 'arvados#repository', true, 'Repository'
+
+ verify_link response_items, 'arvados#virtualMachine', true, 'permission', 'can_login',
+ @vm_uuid, created['uuid'], 'arvados#virtualMachine', false, 'VirtualMachine'
+ end
+
+ test "setup and unsetup user" do
+ authorize_with :admin
+
+ post :setup, {
+ repo_name: 'test_repo',
+ vm_uuid: @vm_uuid,
+ user: {email: 'foo@example.com'},
+ openid_prefix: 'https://www.google.com/accounts/o8/id'
+ }
+
+ assert_response :success
+ response_items = JSON.parse(@response.body)['items']
+ created = find_obj_in_resp response_items, 'User', nil
+ assert_not_nil created['uuid'], 'expected uuid for the new user'
+ assert_equal created['email'], 'foo@example.com', 'expected given email'
+
+ # five extra links: system_group, login, group, repo and vm
+ verify_num_links @all_links_at_start, 5
+
+ verify_link response_items, 'arvados#user', true, 'permission', 'can_login',
+ created['uuid'], created['email'], 'arvados#user', false, 'User'
+
+ verify_link response_items, 'arvados#group', true, 'permission', 'can_read',
+ 'All users', created['uuid'], 'arvados#group', true, 'Group'
+
+ verify_link response_items, 'arvados#repository', true, 'permission', 'can_write',
+ 'test_repo', created['uuid'], 'arvados#repository', true, 'Repository'
+
+ verify_link response_items, 'arvados#virtualMachine', true, 'permission', 'can_login',
+ @vm_uuid, created['uuid'], 'arvados#virtualMachine', false, 'VirtualMachine'
+
+ verify_link_existence created['uuid'], created['email'], true, true, true, true, false
+
+ # now unsetup this user
+ post :unsetup, uuid: created['uuid']
+ assert_response :success
+
+ created2 = JSON.parse(@response.body)
+ assert_not_nil created2['uuid'], 'expected uuid for the newly created user'
+ assert_equal created['uuid'], created2['uuid'], 'expected uuid not found'
+
+ verify_link_existence created['uuid'], created['email'], false, false, false, false, false
+ end
+
+ test "unsetup active user" do
+ authorize_with :active
+ get :current
+ assert_response :success
+ active_user = JSON.parse(@response.body)
+ assert_not_nil active_user['uuid'], 'expected uuid for the active user'
+ assert active_user['is_active'], 'expected is_active for active user'
+ assert active_user['is_invited'], 'expected is_invited for active user'
+
+ verify_link_existence active_user['uuid'], active_user['email'],
+ false, false, false, true, true
+
+ authorize_with :admin
+
+ # now unsetup this user
+ post :unsetup, uuid: active_user['uuid']
+ assert_response :success
+
+ response_user = JSON.parse(@response.body)
+ assert_not_nil response_user['uuid'], 'expected uuid for the upsetup user'
+ assert_equal active_user['uuid'], response_user['uuid'], 'expected uuid not found'
+ assert !response_user['is_active'], 'expected user to be inactive'
+ assert !response_user['is_invited'], 'expected user to be uninvited'
+
+ verify_link_existence response_user['uuid'], response_user['email'],
+ false, false, false, false, false
+ end
+
+ test "setup user with send notification param false and verify no email" do
+ authorize_with :admin
+
+ post :setup, {
+ openid_prefix: 'http://www.example.com/account',
+ send_notification_email: 'false',
+ user: {
+ email: "foo@example.com"
+ }
+ }
+
+ assert_response :success
+ response_items = JSON.parse(@response.body)['items']
+ created = find_obj_in_resp response_items, 'User', nil
+ assert_not_nil created['uuid'], 'expected uuid for the new user'
+ assert_equal created['email'], 'foo@example.com', 'expected given email'
+
+ setup_email = ActionMailer::Base.deliveries.last
+ assert_nil setup_email, 'expected no setup email'
+ end
+
+ test "setup user with send notification param true and verify email" do
+ authorize_with :admin
+
+ post :setup, {
+ openid_prefix: 'http://www.example.com/account',
+ send_notification_email: 'true',
+ user: {
+ email: "foo@example.com"
+ }
+ }
+
+ assert_response :success
+ response_items = JSON.parse(@response.body)['items']
+ created = find_obj_in_resp response_items, 'User', nil
+ assert_not_nil created['uuid'], 'expected uuid for the new user'
+ assert_equal created['email'], 'foo@example.com', 'expected given email'
+
+ setup_email = ActionMailer::Base.deliveries.last
+ assert_not_nil setup_email, 'Expected email after setup'
+
+ assert_equal Rails.configuration.user_notifier_email_from, setup_email.from[0]
+ assert_equal 'foo@example.com', setup_email.to[0]
+ assert_equal 'Welcome to Curoverse', setup_email.subject
+ assert (setup_email.body.to_s.include? 'Your Arvados account has been set up'),
+ 'Expected Your Arvados account has been set up in email body'
+ assert (setup_email.body.to_s.include? 'foo@example.com'),
+ 'Expected user email in email body'
+ assert (setup_email.body.to_s.include? Rails.configuration.workbench_address),
+ 'Expected workbench url in email body'
+ end
+
+ def verify_num_links (original_links, expected_additional_links)
+ links_now = Link.all
+ assert_equal expected_additional_links, Link.all.size-original_links.size,
+ "Expected #{expected_additional_links.inspect} more links"
+ end
+
+ def find_obj_in_resp (response_items, object_type, head_kind=nil)
+ return_obj = nil
+ response_items.each { |x|
+ if !x
+ next
+ end
+
+ if object_type == 'User'
+ if ArvadosModel::resource_class_for_uuid(x['uuid']) == User
+ return_obj = x
+ break
+ end
+ else # looking for a link
+ if x['head_uuid'] and ArvadosModel::resource_class_for_uuid(x['head_uuid']).kind == head_kind
+ return_obj = x
+ break
+ end
+ end
+ }
+ return return_obj
+ end
+
+ def verify_link(response_items, link_object_name, expect_link, link_class,
+ link_name, head_uuid, tail_uuid, head_kind, fetch_object, class_name)
+
+ link = find_obj_in_resp response_items, 'Link', link_object_name
+
+ if !expect_link
+ assert_nil link, "Expected no link for #{link_object_name}"
+ return
+ end
+
+ assert_not_nil link, "Expected link for #{link_object_name}"
+
+ if fetch_object
+ object = Object.const_get(class_name).where(name: head_uuid)
+ assert [] != object, "expected #{class_name} with name #{head_uuid}"
+ head_uuid = object.first[:uuid]
+ end
+ assert_equal link_class, link['link_class'],
+ "did not find expected link_class for #{link_object_name}"
+
+ assert_equal link_name, link['name'],
+ "did not find expected link_name for #{link_object_name}"
+
+ assert_equal tail_uuid, link['tail_uuid'],
+ "did not find expected tail_uuid for #{link_object_name}"
+
+ assert_equal head_kind, link['head_kind'],
+ "did not find expected head_kind for #{link_object_name}"
+
+ assert_equal head_uuid, link['head_uuid'],
+ "did not find expected head_uuid for #{link_object_name}"
+ end
+
+ def verify_link_existence uuid, email, expect_oid_login_perms,
+ expect_repo_perms, expect_vm_perms, expect_group_perms, expect_signatures
+ # verify that all links are deleted for the user
+ oid_login_perms = Link.where(tail_uuid: email,
+ link_class: 'permission',
+ name: 'can_login').where("head_uuid like ?", User.uuid_like_pattern)
+ if expect_oid_login_perms
+ assert oid_login_perms.any?, "expected oid_login_perms"
+ else
+ assert !oid_login_perms.any?, "expected all oid_login_perms deleted"
+ end
+
+ repo_perms = Link.where(tail_uuid: uuid,
+ link_class: 'permission',
+ name: 'can_write').where("head_uuid like ?", Repository.uuid_like_pattern)
+ if expect_repo_perms
+ assert repo_perms.any?, "expected repo_perms"
+ else
+ assert !repo_perms.any?, "expected all repo_perms deleted"
+ end
+
+ vm_login_perms = Link.where(tail_uuid: uuid,
+ link_class: 'permission',
+ name: 'can_login').where("head_uuid like ?", VirtualMachine.uuid_like_pattern)
+ if expect_vm_perms
+ assert vm_login_perms.any?, "expected vm_login_perms"
+ else
+ assert !vm_login_perms.any?, "expected all vm_login_perms deleted"
+ end
+
+ group = Group.where(name: 'All users').select do |g|
+ g[:uuid].match /-f+$/
+ end.first
+ group_read_perms = Link.where(tail_uuid: uuid,
+ head_uuid: group[:uuid],
+ link_class: 'permission',
+ name: 'can_read')
+ if expect_group_perms
+ assert group_read_perms.any?, "expected all users group read perms"
+ else
+ assert !group_read_perms.any?, "expected all users group perm deleted"
+ end
+
+ signed_uuids = Link.where(link_class: 'signature',
+ tail_uuid: uuid)
+
+ if expect_signatures
+ assert signed_uuids.any?, "expected signatures"
+ else
+ assert !signed_uuids.any?, "expected all signatures deleted"
+ end
+
+ end
+
+ def verify_system_group_permission_link_for user_uuid
+ assert_equal 1, Link.where(link_class: 'permission',
+ name: 'can_manage',
+ tail_uuid: system_group_uuid,
+ head_uuid: user_uuid).count
+ end
end
--- /dev/null
+require 'test_helper'
+
+class UserNotifierTest < ActionMailer::TestCase
+ # test "the truth" do
+ # assert true
+ # end
+end
assert_response :success
end
+ test "create token for different user" do
+ post "/arvados/v1/api_client_authorizations", {
+ :format => :json,
+ :api_client_authorization => {
+ :owner_uuid => users(:spectator).uuid
+ }
+ }, {'HTTP_AUTHORIZATION' => "OAuth2 #{api_client_authorizations(:admin_trustedclient).api_token}"}
+ assert_response :success
+
+ get "/arvados/v1/users/current", {
+ :format => :json
+ }, {'HTTP_AUTHORIZATION' => "OAuth2 #{jresponse['api_token']}"}
+ @jresponse = nil
+ assert_equal users(:spectator).uuid, jresponse['uuid']
+ end
+
+ test "refuse to create token for different user if not trusted client" do
+ post "/arvados/v1/api_client_authorizations", {
+ :format => :json,
+ :api_client_authorization => {
+ :owner_uuid => users(:spectator).uuid
+ }
+ }, {'HTTP_AUTHORIZATION' => "OAuth2 #{api_client_authorizations(:admin).api_token}"}
+ assert_response 403
+ end
+
+ test "refuse to create token for different user if not admin" do
+ post "/arvados/v1/api_client_authorizations", {
+ :format => :json,
+ :api_client_authorization => {
+ :owner_uuid => users(:spectator).uuid
+ }
+ }, {'HTTP_AUTHORIZATION' => "OAuth2 #{api_client_authorizations(:active_trustedclient).api_token}"}
+ assert_response 403
+ end
+
end
fixtures :all
test "should get index" do
- get "/arvados/v1/collections", {:format => :json}, {'HTTP_AUTHORIZATION' => "OAuth2 #{api_client_authorizations(:active).api_token}"}
+ get "/arvados/v1/collections", {:format => :json}, auth(:active)
+ assert_response :success
+ assert_equal "arvados#collectionList", jresponse['kind']
+ end
+
+ test "get index with filters= (empty string)" do
+ get "/arvados/v1/collections", {:format => :json, :filters => ''}, auth(:active)
+ assert_response :success
+ assert_equal "arvados#collectionList", jresponse['kind']
+ end
+
+ test "get index with where= (empty string)" do
+ get "/arvados/v1/collections", {:format => :json, :where => ''}, auth(:active)
assert_response :success
assert_equal "arvados#collectionList", jresponse['kind']
end
test "controller 404 response is json" do
- get "/arvados/v1/thingsthatdonotexist", {:format => :xml}, {'HTTP_AUTHORIZATION' => "OAuth2 #{api_client_authorizations(:active).api_token}"}
+ get "/arvados/v1/thingsthatdonotexist", {:format => :xml}, auth(:active)
assert_response 404
assert_equal 1, jresponse['errors'].length
assert_equal true, jresponse['errors'][0].is_a?(String)
end
test "object 404 response is json" do
- get "/arvados/v1/groups/zzzzz-j7d0g-o5ba971173cup4f", {}, {'HTTP_AUTHORIZATION' => "OAuth2 #{api_client_authorizations(:active).api_token}"}
+ get "/arvados/v1/groups/zzzzz-j7d0g-o5ba971173cup4f", {}, auth(:active)
assert_response 404
assert_equal 1, jresponse['errors'].length
assert_equal true, jresponse['errors'][0].is_a?(String)
end
+ test "store collection as json" do
+ post "/arvados/v1/collections", {
+ format: :json,
+ collection: "{\"manifest_text\":\". bad42fa702ae3ea7d888fef11b46f450+44 0:44:md5sum.txt\\n\",\"uuid\":\"ad02e37b6a7f45bbe2ead3c29a109b8a+54\"}"
+ }, auth(:active)
+ assert_response 200
+ assert_equal 'ad02e37b6a7f45bbe2ead3c29a109b8a+54', jresponse['uuid']
+ end
end
--- /dev/null
+require 'test_helper'
+
+class PermissionsTest < ActionDispatch::IntegrationTest
+ fixtures :users, :groups, :api_client_authorizations, :collections
+
+ test "adding and removing direct can_read links" do
+ # try to read collection as spectator
+ get "/arvados/v1/collections/#{collections(:foo_file).uuid}", {:format => :json}, auth(:spectator)
+ assert_response 404
+
+ # try to add permission as spectator
+ post "/arvados/v1/links", {
+ :format => :json,
+ :link => {
+ tail_uuid: users(:spectator).uuid,
+ link_class: 'permission',
+ name: 'can_read',
+ head_uuid: collections(:foo_file).uuid,
+ properties: {}
+ }
+ }, auth(:spectator)
+ assert_response 422
+
+ # add permission as admin
+ post "/arvados/v1/links", {
+ :format => :json,
+ :link => {
+ tail_uuid: users(:spectator).uuid,
+ link_class: 'permission',
+ name: 'can_read',
+ head_uuid: collections(:foo_file).uuid,
+ properties: {}
+ }
+ }, auth(:admin)
+ u = jresponse['uuid']
+ assert_response :success
+
+ # read collection as spectator
+ get "/arvados/v1/collections/#{collections(:foo_file).uuid}", {:format => :json}, auth(:spectator)
+ assert_response :success
+
+ # try to delete permission as spectator
+ delete "/arvados/v1/links/#{u}", {:format => :json}, auth(:spectator)
+ assert_response 403
+
+ # delete permission as admin
+ delete "/arvados/v1/links/#{u}", {:format => :json}, auth(:admin)
+ assert_response :success
+
+ # try to read collection as spectator
+ get "/arvados/v1/collections/#{collections(:foo_file).uuid}", {:format => :json}, auth(:spectator)
+ assert_response 404
+ end
+
+
+ test "adding can_read links from user to group, group to collection" do
+ # try to read collection as spectator
+ get "/arvados/v1/collections/#{collections(:foo_file).uuid}", {:format => :json}, auth(:spectator)
+ assert_response 404
+
+ # add permission for spectator to read group
+ post "/arvados/v1/links", {
+ :format => :json,
+ :link => {
+ tail_uuid: users(:spectator).uuid,
+ link_class: 'permission',
+ name: 'can_read',
+ head_uuid: groups(:private).uuid,
+ properties: {}
+ }
+ }, auth(:admin)
+ assert_response :success
+
+ # try to read collection as spectator
+ get "/arvados/v1/collections/#{collections(:foo_file).uuid}", {:format => :json}, auth(:spectator)
+ assert_response 404
+
+ # add permission for group to read collection
+ post "/arvados/v1/links", {
+ :format => :json,
+ :link => {
+ tail_uuid: groups(:private).uuid,
+ link_class: 'permission',
+ name: 'can_read',
+ head_uuid: collections(:foo_file).uuid,
+ properties: {}
+ }
+ }, auth(:admin)
+ u = jresponse['uuid']
+ assert_response :success
+
+ # try to read collection as spectator
+ get "/arvados/v1/collections/#{collections(:foo_file).uuid}", {:format => :json}, auth(:spectator)
+ assert_response :success
+
+ # delete permission for group to read collection
+ delete "/arvados/v1/links/#{u}", {:format => :json}, auth(:admin)
+ assert_response :success
+
+ # try to read collection as spectator
+ get "/arvados/v1/collections/#{collections(:foo_file).uuid}", {:format => :json}, auth(:spectator)
+ assert_response 404
+
+ end
+
+
+ test "adding can_read links from group to collection, user to group" do
+ # try to read collection as spectator
+ get "/arvados/v1/collections/#{collections(:foo_file).uuid}", {:format => :json}, auth(:spectator)
+ assert_response 404
+
+ # add permission for group to read collection
+ post "/arvados/v1/links", {
+ :format => :json,
+ :link => {
+ tail_uuid: groups(:private).uuid,
+ link_class: 'permission',
+ name: 'can_read',
+ head_uuid: collections(:foo_file).uuid,
+ properties: {}
+ }
+ }, auth(:admin)
+ assert_response :success
+
+ # try to read collection as spectator
+ get "/arvados/v1/collections/#{collections(:foo_file).uuid}", {:format => :json}, auth(:spectator)
+ assert_response 404
+
+ # add permission for spectator to read group
+ post "/arvados/v1/links", {
+ :format => :json,
+ :link => {
+ tail_uuid: users(:spectator).uuid,
+ link_class: 'permission',
+ name: 'can_read',
+ head_uuid: groups(:private).uuid,
+ properties: {}
+ }
+ }, auth(:admin)
+ u = jresponse['uuid']
+ assert_response :success
+
+ # try to read collection as spectator
+ get "/arvados/v1/collections/#{collections(:foo_file).uuid}", {:format => :json}, auth(:spectator)
+ assert_response :success
+
+ # delete permission for spectator to read group
+ delete "/arvados/v1/links/#{u}", {:format => :json}, auth(:admin)
+ assert_response :success
+
+ # try to read collection as spectator
+ get "/arvados/v1/collections/#{collections(:foo_file).uuid}", {:format => :json}, auth(:spectator)
+ assert_response 404
+
+ end
+
+ test "adding can_read links from user to group, group to group, group to collection" do
+ # try to read collection as spectator
+ get "/arvados/v1/collections/#{collections(:foo_file).uuid}", {:format => :json}, auth(:spectator)
+ assert_response 404
+
+ # add permission for user to read group
+ post "/arvados/v1/links", {
+ :format => :json,
+ :link => {
+ tail_uuid: users(:spectator).uuid,
+ link_class: 'permission',
+ name: 'can_read',
+ head_uuid: groups(:private).uuid,
+ properties: {}
+ }
+ }, auth(:admin)
+ assert_response :success
+
+ # add permission for group to read group
+ post "/arvados/v1/links", {
+ :format => :json,
+ :link => {
+ tail_uuid: groups(:private).uuid,
+ link_class: 'permission',
+ name: 'can_read',
+ head_uuid: groups(:empty_lonely_group).uuid,
+ properties: {}
+ }
+ }, auth(:admin)
+ assert_response :success
+
+ # add permission for group to read collection
+ post "/arvados/v1/links", {
+ :format => :json,
+ :link => {
+ tail_uuid: groups(:empty_lonely_group).uuid,
+ link_class: 'permission',
+ name: 'can_read',
+ head_uuid: collections(:foo_file).uuid,
+ properties: {}
+ }
+ }, auth(:admin)
+ u = jresponse['uuid']
+ assert_response :success
+
+ # try to read collection as spectator
+ get "/arvados/v1/collections/#{collections(:foo_file).uuid}", {:format => :json}, auth(:spectator)
+ assert_response :success
+
+ # delete permission for group to read collection
+ delete "/arvados/v1/links/#{u}", {:format => :json}, auth(:admin)
+ assert_response :success
+
+ # try to read collection as spectator
+ get "/arvados/v1/collections/#{collections(:foo_file).uuid}", {:format => :json}, auth(:spectator)
+ assert_response 404
+ end
+
+ test "read-only group-admin sees correct subset of user list" do
+ get "/arvados/v1/users", {:format => :json}, auth(:rominiadmin)
+ assert_response :success
+ resp_uuids = jresponse['items'].collect { |i| i['uuid'] }
+ [[true, users(:rominiadmin).uuid],
+ [true, users(:active).uuid],
+ [false, users(:miniadmin).uuid],
+ [false, users(:spectator).uuid]].each do |should_find, uuid|
+ assert_equal should_find, !resp_uuids.index(uuid).nil?, "rominiadmin should #{'not ' if !should_find}see #{uuid} in user list"
+ end
+ end
+
+ test "read-only group-admin cannot modify administered user" do
+ put "/arvados/v1/users/#{users(:active).uuid}", {
+ :user => {
+ first_name: 'KilroyWasHere'
+ },
+ :format => :json
+ }, auth(:rominiadmin)
+ assert_response 403
+ end
+
+ test "read-only group-admin cannot read or update non-administered user" do
+ get "/arvados/v1/users/#{users(:spectator).uuid}", {
+ :format => :json
+ }, auth(:rominiadmin)
+ assert_response 404
+
+ put "/arvados/v1/users/#{users(:spectator).uuid}", {
+ :user => {
+ first_name: 'KilroyWasHere'
+ },
+ :format => :json
+ }, auth(:rominiadmin)
+ assert_response 404
+ end
+
+ test "RO group-admin finds user's specimens, RW group-admin can update" do
+ [[:rominiadmin, false],
+ [:miniadmin, true]].each do |which_user, update_should_succeed|
+ get "/arvados/v1/specimens", {:format => :json}, auth(which_user)
+ assert_response :success
+ resp_uuids = jresponse['items'].collect { |i| i['uuid'] }
+ [[true, specimens(:owned_by_active_user).uuid],
+ [true, specimens(:owned_by_private_group).uuid],
+ [false, specimens(:owned_by_spectator).uuid],
+ ].each do |should_find, uuid|
+ assert_equal(should_find, !resp_uuids.index(uuid).nil?,
+ "%s should%s see %s in specimen list" %
+ [which_user.to_s,
+ should_find ? '' : 'not ',
+ uuid])
+ put "/arvados/v1/specimens/#{uuid}", {
+ :specimen => {
+ properties: {
+ miniadmin_was_here: true
+ }
+ },
+ :format => :json
+ }, auth(which_user)
+ if !should_find
+ assert_response 404
+ elsif !update_should_succeed
+ assert_response 403
+ else
+ assert_response :success
+ end
+ end
+ end
+ end
+
+end
--- /dev/null
+require 'test_helper'
+
+class ValidLinksTest < ActionDispatch::IntegrationTest
+ fixtures :all
+
+ test "tail must exist on update" do
+ admin_auth = {'HTTP_AUTHORIZATION' => "OAuth2 #{api_client_authorizations(:admin).api_token}"}
+
+ post "/arvados/v1/links", {
+ :format => :json,
+ :link => {
+ link_class: 'test',
+ name: 'stuff',
+ head_uuid: users(:active).uuid,
+ tail_uuid: virtual_machines(:testvm).uuid
+ }
+ }, admin_auth
+ assert_response :success
+ u = jresponse['uuid']
+
+ put "/arvados/v1/links/#{u}", {
+ :format => :json,
+ :link => {
+ tail_uuid: virtual_machines(:testvm2).uuid
+ }
+ }, admin_auth
+ assert_response :success
+ assert_equal virtual_machines(:testvm2).uuid, (ActiveSupport::JSON.decode @response.body)['tail_uuid']
+
+ put "/arvados/v1/links/#{u}", {
+ :format => :json,
+ :link => {
+ tail_uuid: 'zzzzz-tpzed-xyzxyzxerrrorxx'
+ }
+ }, admin_auth
+ assert_response 422
+ end
+
+end
def jresponse
@jresponse ||= ActiveSupport::JSON.decode @response.body
end
+ def auth auth_fixture
+ {'HTTP_AUTHORIZATION' => "OAuth2 #{api_client_authorizations(auth_fixture).api_token}"}
+ end
end
# Ensure permissions are computed from the test fixtures.
--- /dev/null
+require 'test_helper'
+
+class BlobTest < ActiveSupport::TestCase
+ @@api_token = rand(2**512).to_s(36)[0..49]
+ @@key = rand(2**2048).to_s(36)
+ @@blob_data = 'foo'
+ @@blob_locator = Digest::MD5.hexdigest(@@blob_data) +
+ '+' + @@blob_data.size.to_s
+
+ test 'correct' do
+ signed = Blob.sign_locator @@blob_locator, api_token: @@api_token, key: @@key
+ assert_equal true, Blob.verify_signature!(signed, api_token: @@api_token, key: @@key)
+ end
+
+ test 'expired' do
+ signed = Blob.sign_locator @@blob_locator, api_token: @@api_token, key: @@key, ttl: -1
+ assert_raise Blob::InvalidSignatureError do
+ Blob.verify_signature!(signed, api_token: @@api_token, key: @@key)
+ end
+ end
+
+ test 'expired, but no raise' do
+ signed = Blob.sign_locator @@blob_locator, api_token: @@api_token, key: @@key, ttl: -1
+ assert_equal false, Blob.verify_signature(signed,
+ api_token: @@api_token,
+ key: @@key)
+ end
+
+ test 'bogus, wrong block hash' do
+ signed = Blob.sign_locator @@blob_locator, api_token: @@api_token, key: @@key
+ assert_raise Blob::InvalidSignatureError do
+ Blob.verify_signature!(signed.sub('acbd','abcd'), api_token: @@api_token, key: @@key)
+ end
+ end
+
+ test 'bogus, expired' do
+ signed = 'acbd18db4cc2f85cedef654fccc4a4d8+3+Aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa@531641bf'
+ assert_raises Blob::InvalidSignatureError do
+ Blob.verify_signature!(signed, api_token: @@api_token, key: @@key)
+ end
+ end
+
+ test 'bogus, wrong key' do
+ signed = Blob.sign_locator(@@blob_locator,
+ api_token: @@api_token,
+ key: (@@key+'x'))
+ assert_raise Blob::InvalidSignatureError do
+ Blob.verify_signature!(signed, api_token: @@api_token, key: @@key)
+ end
+ end
+
+ test 'bogus, wrong api token' do
+ signed = Blob.sign_locator(@@blob_locator,
+ api_token: @@api_token.reverse,
+ key: @@key)
+ assert_raise Blob::InvalidSignatureError do
+ Blob.verify_signature!(signed, api_token: @@api_token, key: @@key)
+ end
+ end
+
+ test 'bogus, signature format 1' do
+ signed = 'acbd18db4cc2f85cedef654fccc4a4d8+3+Aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa@'
+ assert_raise Blob::InvalidSignatureError do
+ Blob.verify_signature!(signed, api_token: @@api_token, key: @@key)
+ end
+ end
+
+ test 'bogus, signature format 2' do
+ signed = 'acbd18db4cc2f85cedef654fccc4a4d8+3+A@531641bf'
+ assert_raise Blob::InvalidSignatureError do
+ Blob.verify_signature!(signed, api_token: @@api_token, key: @@key)
+ end
+ end
+
+ test 'bogus, signature format 3' do
+ signed = 'acbd18db4cc2f85cedef654fccc4a4d8+3+Axyzzy@531641bf'
+ assert_raise Blob::InvalidSignatureError do
+ Blob.verify_signature!(signed, api_token: @@api_token, key: @@key)
+ end
+ end
+
+ test 'bogus, timestamp format' do
+ signed = 'acbd18db4cc2f85cedef654fccc4a4d8+3+Aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa@xyzzy'
+ assert_raise Blob::InvalidSignatureError do
+ Blob.verify_signature!(signed, api_token: @@api_token, key: @@key)
+ end
+ end
+
+ test 'no signature at all' do
+ assert_raise Blob::InvalidSignatureError do
+ Blob.verify_signature!(@@blob_locator, api_token: @@api_token, key: @@key)
+ end
+ end
+end
require 'test_helper'
class LogTest < ActiveSupport::TestCase
- # test "the truth" do
- # assert true
- # end
+ include CurrentApiClient
+
+ EVENT_TEST_METHODS = {
+ :create => [:created_at, :assert_nil, :assert_not_nil],
+ :update => [:modified_at, :assert_not_nil, :assert_not_nil],
+ :destroy => [nil, :assert_not_nil, :assert_nil],
+ }
+
+ def setup
+ @start_time = Time.now
+ @log_count = 1
+ end
+
+ def assert_properties(test_method, event, props, *keys)
+ verb = (test_method == :assert_nil) ? 'have nil' : 'define'
+ keys.each do |prop_name|
+ assert_includes(props, prop_name, "log properties missing #{prop_name}")
+ self.send(test_method, props[prop_name],
+ "#{event.to_s} log should #{verb} #{prop_name}")
+ end
+ end
+
+ def get_logs_about(thing)
+ Log.where(object_uuid: thing.uuid).order("created_at ASC").all
+ end
+
+ def assert_logged(thing, event_type)
+ logs = get_logs_about(thing)
+ assert_equal(@log_count, logs.size, "log count mismatch")
+ @log_count += 1
+ log = logs.last
+ props = log.properties
+ assert_equal(current_user.andand.uuid, log.owner_uuid,
+ "log is not owned by current user")
+ assert_equal(current_user.andand.uuid, log.modified_by_user_uuid,
+ "log is not 'modified by' current user")
+ assert_equal(current_api_client.andand.uuid, log.modified_by_client_uuid,
+ "log is not 'modified by' current client")
+ assert_equal(thing.uuid, log.object_uuid, "log UUID mismatch")
+ assert_equal(event_type.to_s, log.event_type, "log event type mismatch")
+ time_method, old_props_test, new_props_test = EVENT_TEST_METHODS[event_type]
+ if time_method.nil? or (timestamp = thing.send(time_method)).nil?
+ assert(log.event_at >= @start_time, "log timestamp too old")
+ else
+ assert_in_delta(timestamp, log.event_at, 1, "log timestamp mismatch")
+ end
+ assert_properties(old_props_test, event_type, props,
+ 'old_etag', 'old_attributes')
+ assert_properties(new_props_test, event_type, props,
+ 'new_etag', 'new_attributes')
+ yield props if block_given?
+ end
+
+ def assert_auth_logged_with_clean_properties(auth, event_type)
+ assert_logged(auth, event_type) do |props|
+ ['old_attributes', 'new_attributes'].map { |k| props[k] }.compact
+ .each do |attributes|
+ refute_includes(attributes, 'api_token',
+ "auth log properties include sensitive API token")
+ end
+ yield props if block_given?
+ end
+ end
+
+ def set_user_from_auth(auth_name)
+ client_auth = api_client_authorizations(auth_name)
+ Thread.current[:api_client_authorization] = client_auth
+ Thread.current[:api_client] = client_auth.api_client
+ Thread.current[:user] = client_auth.user
+ end
+
+ test "creating a user makes a log" do
+ set_user_from_auth :admin_trustedclient
+ u = User.new(first_name: "Log", last_name: "Test")
+ u.save!
+ assert_logged(u, :create) do |props|
+ assert_equal(u.etag, props['new_etag'], "new user etag mismatch")
+ assert_equal(u.first_name, props['new_attributes']['first_name'],
+ "new user first name mismatch")
+ assert_equal(u.last_name, props['new_attributes']['last_name'],
+ "new user first name mismatch")
+ end
+ end
+
+ test "updating a virtual machine makes a log" do
+ set_user_from_auth :admin_trustedclient
+ vm = virtual_machines(:testvm)
+ orig_etag = vm.etag
+ vm.hostname = 'testvm.testshell'
+ vm.save!
+ assert_logged(vm, :update) do |props|
+ assert_equal(orig_etag, props['old_etag'], "updated VM old etag mismatch")
+ assert_equal(vm.etag, props['new_etag'], "updated VM new etag mismatch")
+ assert_equal('testvm.shell', props['old_attributes']['hostname'],
+ "updated VM old name mismatch")
+ assert_equal('testvm.testshell', props['new_attributes']['hostname'],
+ "updated VM new name mismatch")
+ end
+ end
+
+ test "destroying an authorization makes a log" do
+ set_user_from_auth :admin_trustedclient
+ auth = api_client_authorizations(:spectator)
+ orig_etag = auth.etag
+ orig_attrs = auth.attributes
+ orig_attrs.delete 'api_token'
+ auth.destroy
+ assert_logged(auth, :destroy) do |props|
+ assert_equal(orig_etag, props['old_etag'], "destroyed auth etag mismatch")
+ assert_equal(orig_attrs, props['old_attributes'],
+ "destroyed auth attributes mismatch")
+ end
+ end
+
+ test "saving an unchanged client still makes a log" do
+ set_user_from_auth :admin_trustedclient
+ client = api_clients(:untrusted)
+ client.is_trusted = client.is_trusted
+ client.save!
+ assert_logged(client, :update) do |props|
+ ['old', 'new'].each do |age|
+ assert_equal(client.etag, props["#{age}_etag"],
+ "unchanged client #{age} etag mismatch")
+ assert_equal(client.attributes, props["#{age}_attributes"],
+ "unchanged client #{age} attributes mismatch")
+ end
+ end
+ end
+
+ test "updating a group twice makes two logs" do
+ set_user_from_auth :admin_trustedclient
+ group = groups(:empty_lonely_group)
+ name1 = group.name
+ name2 = "#{name1} under test"
+ group.name = name2
+ group.save!
+ assert_logged(group, :update) do |props|
+ assert_equal(name1, props['old_attributes']['name'],
+ "group start name mismatch")
+ assert_equal(name2, props['new_attributes']['name'],
+ "group updated name mismatch")
+ end
+ group.name = name1
+ group.save!
+ assert_logged(group, :update) do |props|
+ assert_equal(name2, props['old_attributes']['name'],
+ "group pre-revert name mismatch")
+ assert_equal(name1, props['new_attributes']['name'],
+ "group final name mismatch")
+ end
+ end
+
+ test "making a log doesn't get logged" do
+ set_user_from_auth :active_trustedclient
+ log = Log.new
+ log.save!
+ assert_equal(0, get_logs_about(log).size, "made a Log about a Log")
+ end
+
+ test "non-admins can't modify or delete logs" do
+ set_user_from_auth :active_trustedclient
+ log = Log.new(summary: "immutable log test")
+ assert_nothing_raised { log.save! }
+ log.summary = "log mutation test should fail"
+ assert_raise(ArvadosModel::PermissionDeniedError) { log.save! }
+ assert_raise(ArvadosModel::PermissionDeniedError) { log.destroy }
+ end
+
+ test "admins can modify and delete logs" do
+ set_user_from_auth :admin_trustedclient
+ log = Log.new(summary: "admin log mutation test")
+ assert_nothing_raised { log.save! }
+ log.summary = "admin mutated log test"
+ assert_nothing_raised { log.save! }
+ assert_nothing_raised { log.destroy }
+ end
+
+ test "failure saving log causes failure saving object" do
+ Log.class_eval do
+ alias_method :_orig_validations, :perform_validations
+ def perform_validations(options)
+ false
+ end
+ end
+ begin
+ set_user_from_auth :active_trustedclient
+ user = users(:active)
+ user.first_name = 'Test'
+ assert_raise(ActiveRecord::RecordInvalid) { user.save! }
+ ensure
+ Log.class_eval do
+ alias_method :perform_validations, :_orig_validations
+ end
+ end
+ end
+
+ test "don't log changes only to ApiClientAuthorization.last_used_*" do
+ set_user_from_auth :admin_trustedclient
+ auth = api_client_authorizations(:spectator)
+ start_log_count = get_logs_about(auth).size
+ auth.last_used_at = Time.now
+ auth.last_used_by_ip_address = '::1'
+ auth.save!
+ assert_equal(start_log_count, get_logs_about(auth).size,
+ "log count changed after 'using' ApiClientAuthorization")
+ auth.created_by_ip_address = '::1'
+ auth.save!
+ assert_logged(auth, :update)
+ end
+
+ test "token isn't included in ApiClientAuthorization logs" do
+ set_user_from_auth :admin_trustedclient
+ auth = ApiClientAuthorization.new
+ auth.user = users(:spectator)
+ auth.api_client = api_clients(:untrusted)
+ auth.save!
+ assert_auth_logged_with_clean_properties(auth, :create)
+ auth.expires_at = Time.now
+ auth.save!
+ assert_auth_logged_with_clean_properties(auth, :update)
+ auth.destroy
+ assert_auth_logged_with_clean_properties(auth, :destroy)
+ end
end
--- /dev/null
+require 'test_helper'
+
+class UserNotifierTest < ActionMailer::TestCase
+
+ # Send the email, then test that it got queued
+ test "account is setup" do
+ user = users :active
+ email = UserNotifier.account_is_setup user
+
+ assert_not_nil email
+
+ # Test the body of the sent email contains what we expect it to
+ assert_equal Rails.configuration.user_notifier_email_from, email.from.first
+ assert_equal user.email, email.to.first
+ assert_equal 'Welcome to Curoverse', email.subject
+ assert (email.body.to_s.include? 'Your Arvados account has been set up'),
+ 'Expected Your Arvados account has been set up in email body'
+ assert (email.body.to_s.include? user.email),
+ 'Expected user email in email body'
+ assert (email.body.to_s.include? Rails.configuration.workbench_address),
+ 'Expected workbench url in email body'
+ end
+
+end
require 'test_helper'
class UserTest < ActiveSupport::TestCase
- # test "the truth" do
- # assert true
- # end
+ include CurrentApiClient
+
+ # The fixture services/api/test/fixtures/users.yml serves as the input for this test case
+ setup do
+ # Make sure system_user exists before making "pre-test users" list
+ system_user
+
+ @all_users = User.find(:all)
+
+ @all_users.each do |user|
+ if user.uuid == system_user_uuid
+ @system_user = user
+ elsif user.is_admin && user.is_active
+ @admin_user = user
+ elsif user.is_active && !user.is_admin
+ @active_user = user
+ elsif !user.is_active && !user.is_invited
+ @uninvited_user = user
+ end
+ end
+ end
+
+ test "check non-admin active user properties" do
+ assert !@active_user.is_admin, 'is_admin should not be set for a non-admin user'
+ assert @active_user.is_active, 'user should be active'
+ assert @active_user.is_invited, 'is_invited should be set'
+ assert_not_nil @active_user.prefs, "user's preferences should be non-null, but may be size zero"
+ assert (@active_user.can? :read=>"#{@active_user.uuid}"), "user should be able to read own object"
+ assert (@active_user.can? :write=>"#{@active_user.uuid}"), "user should be able to write own object"
+ assert (@active_user.can? :manage=>"#{@active_user.uuid}"), "user should be able to manage own object"
+
+ assert @active_user.groups_i_can(:read).size > 0, "active user should be able read at least one group"
+
+ # non-admin user cannot manage or write other user objects
+ assert !(@active_user.can? :read=>"#{@uninvited_user.uuid}")
+ assert !(@active_user.can? :write=>"#{@uninvited_user.uuid}")
+ assert !(@active_user.can? :manage=>"#{@uninvited_user.uuid}")
+ end
+
+ test "check admin user properties" do
+ assert @admin_user.is_admin, 'is_admin should be set for admin user'
+ assert @admin_user.is_active, 'admin user cannot be inactive'
+ assert @admin_user.is_invited, 'is_invited should be set'
+ assert_not_nil @admin_user.uuid.size, "user's uuid should be non-null"
+ assert_not_nil @admin_user.prefs, "user's preferences should be non-null, but may be size zero"
+ assert @admin_user.identity_url.size > 0, "user's identity url is expected"
+ assert @admin_user.can? :read=>"#{@admin_user.uuid}"
+ assert @admin_user.can? :write=>"#{@admin_user.uuid}"
+ assert @admin_user.can? :manage=>"#{@admin_user.uuid}"
+
+ assert @admin_user.groups_i_can(:read).size > 0, "admin active user should be able read at least one group"
+ assert @admin_user.groups_i_can(:write).size > 0, "admin active user should be able write to at least one group"
+ assert @admin_user.groups_i_can(:manage).size > 0, "admin active user should be able manage at least one group"
+
+ # admin user can also write or manage other users
+ assert @admin_user.can? :read=>"#{@uninvited_user.uuid}"
+ assert @admin_user.can? :write=>"#{@uninvited_user.uuid}"
+ assert @admin_user.can? :manage=>"#{@uninvited_user.uuid}"
+ end
+
+ test "check inactive and uninvited user properties" do
+ assert !@uninvited_user.is_admin, 'is_admin should not be set for a non-admin user'
+ assert !@uninvited_user.is_active, 'user should be inactive'
+ assert !@uninvited_user.is_invited, 'is_invited should not be set'
+ assert @uninvited_user.can? :read=>"#{@uninvited_user.uuid}"
+ assert @uninvited_user.can? :write=>"#{@uninvited_user.uuid}"
+ assert @uninvited_user.can? :manage=>"#{@uninvited_user.uuid}"
+
+ assert @uninvited_user.groups_i_can(:read).size == 0, "inactive and uninvited user should not be able read any groups"
+ assert @uninvited_user.groups_i_can(:write).size == 0, "inactive and uninvited user should not be able write to any groups"
+ assert @uninvited_user.groups_i_can(:manage).size == 0, "inactive and uninvited user should not be able manage any groups"
+ end
+
+ test "find user method checks" do
+ User.find(:all).each do |user|
+ assert_not_nil user.uuid, "non-null uuid expected for " + user.full_name
+ end
+
+ user = users(:active) # get the active user
+
+ found_user = User.find(user.id) # find a user by the row id
+
+ assert_equal found_user.full_name, user.first_name + ' ' + user.last_name
+ assert_equal found_user.identity_url, user.identity_url
+ end
+
+ test "create new user" do
+ Thread.current[:user] = @admin_user # set admin user as the current user
+
+ user = User.new
+ user.first_name = "first_name_for_newly_created_user"
+ user.save
+
+ # verify there is one extra user in the db now
+ assert_equal @all_users.size+1, User.find(:all).size
+
+ user = User.find(user.id) # get the user back
+ assert_equal(user.first_name, 'first_name_for_newly_created_user')
+ assert_not_nil user.uuid, 'uuid should be set for newly created user'
+ assert_nil user.email, 'email should be null for newly created user, because it was not passed in'
+ assert_nil user.identity_url, 'identity_url should be null for newly created user, because it was not passed in'
+
+ user.first_name = 'first_name_for_newly_created_user_updated'
+ user.save
+ user = User.find(user.id) # get the user back
+ assert_equal(user.first_name, 'first_name_for_newly_created_user_updated')
+ end
+
+ test "update existing user" do
+ Thread.current[:user] = @active_user # set active user as current user
+ @active_user.first_name = "first_name_changed"
+ @active_user.save
+
+ @active_user = User.find(@active_user.id) # get the user back
+ assert_equal(@active_user.first_name, 'first_name_changed')
+
+ # admin user also should be able to update the "active" user info
+ Thread.current[:user] = @admin_user # set admin user as current user
+ @active_user.first_name = "first_name_changed_by_admin_for_active_user"
+ @active_user.save
+
+ @active_user = User.find(@active_user.id) # get the user back
+ assert_equal(@active_user.first_name, 'first_name_changed_by_admin_for_active_user')
+ end
+
+ test "delete a user and verify" do
+ active_user_uuid = @active_user.uuid
+
+ Thread.current[:user] = @admin_user
+ @active_user.delete
+
+ found_deleted_user = false
+ User.find(:all).each do |user|
+ if user.uuid == active_user_uuid
+ found_deleted_user = true
+ break
+ end
+ end
+ assert !found_deleted_user, "found deleted user: "+active_user_uuid
+
+ end
+
+ test "create new user as non-admin user" do
+ Thread.current[:user] = @active_user
+
+ begin
+ user = User.new
+ user.save
+ rescue ArvadosModel::PermissionDeniedError => e
+ end
+ assert (e.message.include? 'PermissionDeniedError'),
+ 'Expected PermissionDeniedError'
+ end
+
+ test "setup new user" do
+ Thread.current[:user] = @admin_user
+
+ email = 'foo@example.com'
+ openid_prefix = 'http://openid/prefix'
+
+ user = User.create ({uuid: 'zzzzz-tpzed-abcdefghijklmno', email: email})
+
+ vm = VirtualMachine.create
+
+ response = User.setup user, openid_prefix, 'test_repo', vm.uuid
+
+ resp_user = find_obj_in_resp response, 'User'
+ verify_user resp_user, email
+
+ oid_login_perm = find_obj_in_resp response, 'Link', 'arvados#user'
+
+ verify_link oid_login_perm, 'permission', 'can_login', resp_user[:email],
+ resp_user[:uuid]
+
+ assert_equal openid_prefix, oid_login_perm[:properties][:identity_url_prefix],
+ 'expected identity_url_prefix not found for oid_login_perm'
+
+ group_perm = find_obj_in_resp response, 'Link', 'arvados#group'
+ verify_link group_perm, 'permission', 'can_read', resp_user[:uuid], nil
+
+ repo_perm = find_obj_in_resp response, 'Link', 'arvados#repository'
+ verify_link repo_perm, 'permission', 'can_write', resp_user[:uuid], nil
+
+ vm_perm = find_obj_in_resp response, 'Link', 'arvados#virtualMachine'
+ verify_link vm_perm, 'permission', 'can_login', resp_user[:uuid], vm.uuid
+ end
+
+ test "setup new user in multiple steps" do
+ Thread.current[:user] = @admin_user
+
+ email = 'foo@example.com'
+ openid_prefix = 'http://openid/prefix'
+
+ user = User.create ({uuid: 'zzzzz-tpzed-abcdefghijklmno', email: email})
+
+ response = User.setup user, openid_prefix
+
+ resp_user = find_obj_in_resp response, 'User'
+ verify_user resp_user, email
+
+ oid_login_perm = find_obj_in_resp response, 'Link', 'arvados#user'
+ verify_link oid_login_perm, 'permission', 'can_login', resp_user[:email],
+ resp_user[:uuid]
+ assert_equal openid_prefix, oid_login_perm[:properties][:identity_url_prefix],
+ 'expected identity_url_prefix not found for oid_login_perm'
+
+ group_perm = find_obj_in_resp response, 'Link', 'arvados#group'
+ verify_link group_perm, 'permission', 'can_read', resp_user[:uuid], nil
+
+ # invoke setup again with repo_name
+ response = User.setup user, openid_prefix, 'test_repo'
+ resp_user = find_obj_in_resp response, 'User', nil
+ verify_user resp_user, email
+ assert_equal user.uuid, resp_user[:uuid], 'expected uuid not found'
+
+ group_perm = find_obj_in_resp response, 'Link', 'arvados#group'
+ verify_link group_perm, 'permission', 'can_read', resp_user[:uuid], nil
+
+ repo_perm = find_obj_in_resp response, 'Link', 'arvados#repository'
+ verify_link repo_perm, 'permission', 'can_write', resp_user[:uuid], nil
+
+ # invoke setup again with a vm_uuid
+ vm = VirtualMachine.create
+
+ response = User.setup user, openid_prefix, 'test_repo', vm.uuid
+
+ resp_user = find_obj_in_resp response, 'User', nil
+ verify_user resp_user, email
+ assert_equal user.uuid, resp_user[:uuid], 'expected uuid not found'
+
+ group_perm = find_obj_in_resp response, 'Link', 'arvados#group'
+ verify_link group_perm, 'permission', 'can_read', resp_user[:uuid], nil
+
+ repo_perm = find_obj_in_resp response, 'Link', 'arvados#repository'
+ verify_link repo_perm, 'permission', 'can_write', resp_user[:uuid], nil
+
+ vm_perm = find_obj_in_resp response, 'Link', 'arvados#virtualMachine'
+ verify_link vm_perm, 'permission', 'can_login', resp_user[:uuid], vm.uuid
+ end
+
+ def find_obj_in_resp (response_items, object_type, head_kind=nil)
+ return_obj = nil
+ response_items.each { |x|
+ if !x
+ next
+ end
+
+ if object_type == 'User'
+ if ArvadosModel::resource_class_for_uuid(x['uuid']) == User
+ return_obj = x
+ break
+ end
+ else # looking for a link
+ if ArvadosModel::resource_class_for_uuid(x['head_uuid']).kind == head_kind
+ return_obj = x
+ break
+ end
+ end
+ }
+ return return_obj
+ end
+
+ def verify_user (resp_user, email)
+ assert_not_nil resp_user, 'expected user object'
+ assert_not_nil resp_user['uuid'], 'expected user object'
+ assert_equal email, resp_user['email'], 'expected email not found'
+
+ end
+
+ def verify_link (link_object, link_class, link_name, tail_uuid, head_uuid)
+ assert_not_nil link_object, "expected link for #{link_class} #{link_name}"
+ assert_not_nil link_object[:uuid],
+ "expected non-nil uuid for link for #{link_class} #{link_name}"
+ assert_equal link_class, link_object[:link_class],
+ "expected link_class not found for #{link_class} #{link_name}"
+ assert_equal link_name, link_object[:name],
+ "expected link_name not found for #{link_class} #{link_name}"
+ assert_equal tail_uuid, link_object[:tail_uuid],
+ "expected tail_uuid not found for #{link_class} #{link_name}"
+ if head_uuid
+ assert_equal head_uuid, link_object[:head_uuid],
+ "expected head_uuid not found for #{link_class} #{link_name}"
+ end
+ end
+
end
--- /dev/null
+package main
+
+import (
+ "bufio"
+ "bytes"
+ "crypto/md5"
+ "errors"
+ "flag"
+ "fmt"
+ "github.com/gorilla/mux"
+ "io"
+ "io/ioutil"
+ "log"
+ "net/http"
+ "os"
+ "strconv"
+ "strings"
+ "syscall"
+ "time"
+)
+
+// ======================
+// Configuration settings
+//
+// TODO(twp): make all of these configurable via command line flags
+// and/or configuration file settings.
+
+// Default TCP address on which to listen for requests.
+const DEFAULT_ADDR = ":25107"
+
+// A Keep "block" is 64MB.
+const BLOCKSIZE = 64 * 1024 * 1024
+
+// A Keep volume must have at least MIN_FREE_KILOBYTES available
+// in order to permit writes.
+const MIN_FREE_KILOBYTES = BLOCKSIZE / 1024
+
+var PROC_MOUNTS = "/proc/mounts"
+
+var KeepVolumes []string
+
+// ==========
+// Error types.
+//
+type KeepError struct {
+ HTTPCode int
+ ErrMsg string
+}
+
+var (
+ CollisionError = &KeepError{400, "Collision"}
+ MD5Error = &KeepError{401, "MD5 Failure"}
+ CorruptError = &KeepError{402, "Corruption"}
+ NotFoundError = &KeepError{404, "Not Found"}
+ GenericError = &KeepError{500, "Fail"}
+ FullError = &KeepError{503, "Full"}
+ TooLongError = &KeepError{504, "Too Long"}
+)
+
+func (e *KeepError) Error() string {
+ return e.ErrMsg
+}
+
+// This error is returned by ReadAtMost if the available
+// data exceeds BLOCKSIZE bytes.
+var ReadErrorTooLong = errors.New("Too long")
+
+func main() {
+ // Parse command-line flags:
+ //
+ // -listen=ipaddr:port
+ // Interface on which to listen for requests. Use :port without
+ // an ipaddr to listen on all network interfaces.
+ // Examples:
+ // -listen=127.0.0.1:4949
+ // -listen=10.0.1.24:8000
+ // -listen=:25107 (to listen to port 25107 on all interfaces)
+ //
+ // -volumes
+ // A comma-separated list of directories to use as Keep volumes.
+ // Example:
+ // -volumes=/var/keep01,/var/keep02,/var/keep03/subdir
+ //
+ // If -volumes is empty or is not present, Keep will select volumes
+ // by looking at currently mounted filesystems for /keep top-level
+ // directories.
+
+ var listen, keepvols string
+ flag.StringVar(&listen, "listen", DEFAULT_ADDR,
+ "interface on which to listen for requests, in the format ipaddr:port. e.g. -listen=10.0.1.24:8000. Use -listen=:port to listen on all network interfaces.")
+ flag.StringVar(&keepvols, "volumes", "",
+ "Comma-separated list of directories to use for Keep volumes, e.g. -volumes=/var/keep1,/var/keep2. If empty or not supplied, Keep will scan mounted filesystems for volumes with a /keep top-level directory.")
+ flag.Parse()
+
+ // Look for local keep volumes.
+ if keepvols == "" {
+ // TODO(twp): decide whether this is desirable default behavior.
+ // In production we may want to require the admin to specify
+ // Keep volumes explicitly.
+ KeepVolumes = FindKeepVolumes()
+ } else {
+ KeepVolumes = strings.Split(keepvols, ",")
+ }
+
+ if len(KeepVolumes) == 0 {
+ log.Fatal("could not find any keep volumes")
+ }
+ for _, v := range KeepVolumes {
+ log.Println("keep volume:", v)
+ }
+
+ // Set up REST handlers.
+ //
+ // Start with a router that will route each URL path to an
+ // appropriate handler.
+ //
+ rest := mux.NewRouter()
+ rest.HandleFunc("/{hash:[0-9a-f]{32}}", GetBlockHandler).Methods("GET")
+ rest.HandleFunc("/{hash:[0-9a-f]{32}}", PutBlockHandler).Methods("PUT")
+
+ // Tell the built-in HTTP server to direct all requests to the REST
+ // router.
+ http.Handle("/", rest)
+
+ // Start listening for requests.
+ http.ListenAndServe(listen, nil)
+}
+
+// FindKeepVolumes
+// Returns a list of Keep volumes mounted on this system.
+//
+// A Keep volume is a normal or tmpfs volume with a /keep
+// directory at the top level of the mount point.
+//
+func FindKeepVolumes() []string {
+ vols := make([]string, 0)
+
+ if f, err := os.Open(PROC_MOUNTS); err != nil {
+ log.Fatalf("opening %s: %s\n", PROC_MOUNTS, err)
+ } else {
+ scanner := bufio.NewScanner(f)
+ for scanner.Scan() {
+ args := strings.Fields(scanner.Text())
+ dev, mount := args[0], args[1]
+ if (dev == "tmpfs" || strings.HasPrefix(dev, "/dev/")) && mount != "/" {
+ keep := mount + "/keep"
+ if st, err := os.Stat(keep); err == nil && st.IsDir() {
+ vols = append(vols, keep)
+ }
+ }
+ }
+ if err := scanner.Err(); err != nil {
+ log.Fatal(err)
+ }
+ }
+ return vols
+}
+
+func GetBlockHandler(w http.ResponseWriter, req *http.Request) {
+ hash := mux.Vars(req)["hash"]
+
+ block, err := GetBlock(hash)
+ if err != nil {
+ http.Error(w, err.Error(), 404)
+ return
+ }
+
+ _, err = w.Write(block)
+ if err != nil {
+ log.Printf("GetBlockHandler: writing response: %s", err)
+ }
+
+ return
+}
+
+func PutBlockHandler(w http.ResponseWriter, req *http.Request) {
+ hash := mux.Vars(req)["hash"]
+
+ // Read the block data to be stored.
+ // If the request exceeds BLOCKSIZE bytes, issue a HTTP 500 error.
+ //
+ // Note: because req.Body is a buffered Reader, each Read() call will
+ // collect only the data in the network buffer (typically 16384 bytes),
+ // even if it is passed a much larger slice.
+ //
+ // Instead, call ReadAtMost to read data from the socket
+ // repeatedly until either EOF or BLOCKSIZE bytes have been read.
+ //
+ if buf, err := ReadAtMost(req.Body, BLOCKSIZE); err == nil {
+ if err := PutBlock(buf, hash); err == nil {
+ w.WriteHeader(http.StatusOK)
+ } else {
+ ke := err.(*KeepError)
+ http.Error(w, ke.Error(), ke.HTTPCode)
+ }
+ } else {
+ log.Println("error reading request: ", err)
+ errmsg := err.Error()
+ if err == ReadErrorTooLong {
+ // Use a more descriptive error message that includes
+ // the maximum request size.
+ errmsg = fmt.Sprintf("Max request size %d bytes", BLOCKSIZE)
+ }
+ http.Error(w, errmsg, 500)
+ }
+}
+
+func GetBlock(hash string) ([]byte, error) {
+ var buf = make([]byte, BLOCKSIZE)
+
+ // Attempt to read the requested hash from a keep volume.
+ for _, vol := range KeepVolumes {
+ var f *os.File
+ var err error
+ var nread int
+
+ blockFilename := fmt.Sprintf("%s/%s/%s", vol, hash[0:3], hash)
+
+ f, err = os.Open(blockFilename)
+ if err != nil {
+ if !os.IsNotExist(err) {
+ // A block is stored on only one Keep disk,
+ // so os.IsNotExist is expected. Report any other errors.
+ log.Printf("%s: opening %s: %s\n", vol, blockFilename, err)
+ }
+ continue
+ }
+
+ nread, err = f.Read(buf)
+ if err != nil {
+ log.Printf("%s: reading %s: %s\n", vol, blockFilename, err)
+ continue
+ }
+
+ // Double check the file checksum.
+ //
+ filehash := fmt.Sprintf("%x", md5.Sum(buf[:nread]))
+ if filehash != hash {
+ // TODO(twp): this condition probably represents a bad disk and
+ // should raise major alarm bells for an administrator: e.g.
+ // they should be sent directly to an event manager at high
+ // priority or logged as urgent problems.
+ //
+ log.Printf("%s: checksum mismatch: %s (actual hash %s)\n",
+ vol, blockFilename, filehash)
+ return buf, CorruptError
+ }
+
+ // Success!
+ return buf[:nread], nil
+ }
+
+ log.Printf("%s: not found on any volumes, giving up\n", hash)
+ return buf, NotFoundError
+}
+
+/* PutBlock(block, hash)
+ Stores the BLOCK (identified by the content id HASH) in Keep.
+
+ The MD5 checksum of the block must be identical to the content id HASH.
+ If not, an error is returned.
+
+ PutBlock stores the BLOCK on the first Keep volume with free space.
+ A failure code is returned to the user only if all volumes fail.
+
+ On success, PutBlock returns nil.
+ On failure, it returns a KeepError with one of the following codes:
+
+ 400 Collision
+ A different block with the same hash already exists on this
+ Keep server.
+ 401 MD5Fail
+ The MD5 hash of the BLOCK does not match the argument HASH.
+ 503 Full
+ There was not enough space left in any Keep volume to store
+ the object.
+ 500 Fail
+ The object could not be stored for some other reason (e.g.
+ all writes failed). The text of the error message should
+ provide as much detail as possible.
+*/
+
+func PutBlock(block []byte, hash string) error {
+ // Check that BLOCK's checksum matches HASH.
+ blockhash := fmt.Sprintf("%x", md5.Sum(block))
+ if blockhash != hash {
+ log.Printf("%s: MD5 checksum %s did not match request", hash, blockhash)
+ return MD5Error
+ }
+
+ // If we already have a block on disk under this identifier, return
+ // success (but check for MD5 collisions).
+ // The only errors that GetBlock can return are ErrCorrupt and ErrNotFound.
+ // In either case, we want to write our new (good) block to disk, so there is
+ // nothing special to do if err != nil.
+ if oldblock, err := GetBlock(hash); err == nil {
+ if bytes.Compare(block, oldblock) == 0 {
+ return nil
+ } else {
+ return CollisionError
+ }
+ }
+
+ // Store the block on the first available Keep volume.
+ allFull := true
+ for _, vol := range KeepVolumes {
+ if IsFull(vol) {
+ continue
+ }
+ allFull = false
+ blockDir := fmt.Sprintf("%s/%s", vol, hash[0:3])
+ if err := os.MkdirAll(blockDir, 0755); err != nil {
+ log.Printf("%s: could not create directory %s: %s",
+ hash, blockDir, err)
+ continue
+ }
+
+ tmpfile, tmperr := ioutil.TempFile(blockDir, "tmp"+hash)
+ if tmperr != nil {
+ log.Printf("ioutil.TempFile(%s, tmp%s): %s", blockDir, hash, tmperr)
+ continue
+ }
+ blockFilename := fmt.Sprintf("%s/%s", blockDir, hash)
+
+ if _, err := tmpfile.Write(block); err != nil {
+ log.Printf("%s: writing to %s: %s\n", vol, blockFilename, err)
+ continue
+ }
+ if err := tmpfile.Close(); err != nil {
+ log.Printf("closing %s: %s\n", tmpfile.Name(), err)
+ os.Remove(tmpfile.Name())
+ continue
+ }
+ if err := os.Rename(tmpfile.Name(), blockFilename); err != nil {
+ log.Printf("rename %s %s: %s\n", tmpfile.Name(), blockFilename, err)
+ os.Remove(tmpfile.Name())
+ continue
+ }
+ return nil
+ }
+
+ if allFull {
+ log.Printf("all Keep volumes full")
+ return FullError
+ } else {
+ log.Printf("all Keep volumes failed")
+ return GenericError
+ }
+}
+
+func IsFull(volume string) (isFull bool) {
+ fullSymlink := volume + "/full"
+
+ // Check if the volume has been marked as full in the last hour.
+ if link, err := os.Readlink(fullSymlink); err == nil {
+ if ts, err := strconv.Atoi(link); err == nil {
+ fulltime := time.Unix(int64(ts), 0)
+ if time.Since(fulltime).Hours() < 1.0 {
+ return true
+ }
+ }
+ }
+
+ if avail, err := FreeDiskSpace(volume); err == nil {
+ isFull = avail < MIN_FREE_KILOBYTES
+ } else {
+ log.Printf("%s: FreeDiskSpace: %s\n", volume, err)
+ isFull = false
+ }
+
+ // If the volume is full, timestamp it.
+ if isFull {
+ now := fmt.Sprintf("%d", time.Now().Unix())
+ os.Symlink(now, fullSymlink)
+ }
+ return
+}
+
+// FreeDiskSpace(volume)
+// Returns the amount of available disk space on VOLUME,
+// as a number of 1k blocks.
+//
+func FreeDiskSpace(volume string) (free uint64, err error) {
+ var fs syscall.Statfs_t
+ err = syscall.Statfs(volume, &fs)
+ if err == nil {
+ // Statfs output is not guaranteed to measure free
+ // space in terms of 1K blocks.
+ free = fs.Bavail * uint64(fs.Bsize) / 1024
+ }
+
+ return
+}
+
+// ReadAtMost
+// Reads bytes repeatedly from an io.Reader until either
+// encountering EOF, or the maxbytes byte limit has been reached.
+// Returns a byte slice of the bytes that were read.
+//
+// If the reader contains more than maxbytes, returns a nil slice
+// and an error.
+//
+func ReadAtMost(r io.Reader, maxbytes int) ([]byte, error) {
+ // Attempt to read one more byte than maxbytes.
+ lr := io.LimitReader(r, int64(maxbytes+1))
+ buf, err := ioutil.ReadAll(lr)
+ if len(buf) > maxbytes {
+ return nil, ReadErrorTooLong
+ }
+ return buf, err
+}
--- /dev/null
+package main
+
+import (
+ "bytes"
+ "fmt"
+ "io/ioutil"
+ "os"
+ "path"
+ "testing"
+)
+
+var TEST_BLOCK = []byte("The quick brown fox jumps over the lazy dog.")
+var TEST_HASH = "e4d909c290d0fb1ca068ffaddf22cbd0"
+var BAD_BLOCK = []byte("The magic words are squeamish ossifrage.")
+
+// TODO(twp): Tests still to be written
+//
+// * PutBlockFull
+// - test that PutBlock returns 503 Full if the filesystem is full.
+// (must mock FreeDiskSpace or Statfs? use a tmpfs?)
+//
+// * PutBlockWriteErr
+// - test the behavior when Write returns an error.
+// - Possible solutions: use a small tmpfs and a high
+// MIN_FREE_KILOBYTES to trick PutBlock into attempting
+// to write a block larger than the amount of space left
+// - use an interface to mock ioutil.TempFile with a File
+// object that always returns an error on write
+//
+// ========================================
+// GetBlock tests.
+// ========================================
+
+// TestGetBlock
+// Test that simple block reads succeed.
+//
+func TestGetBlock(t *testing.T) {
+ defer teardown()
+
+ // Prepare two test Keep volumes. Our block is stored on the second volume.
+ KeepVolumes = setup(t, 2)
+ store(t, KeepVolumes[1], TEST_HASH, TEST_BLOCK)
+
+ // Check that GetBlock returns success.
+ result, err := GetBlock(TEST_HASH)
+ if err != nil {
+ t.Errorf("GetBlock error: %s", err)
+ }
+ if fmt.Sprint(result) != fmt.Sprint(TEST_BLOCK) {
+ t.Errorf("expected %s, got %s", TEST_BLOCK, result)
+ }
+}
+
+// TestGetBlockMissing
+// GetBlock must return an error when the block is not found.
+//
+func TestGetBlockMissing(t *testing.T) {
+ defer teardown()
+
+ // Create two empty test Keep volumes.
+ KeepVolumes = setup(t, 2)
+
+ // Check that GetBlock returns failure.
+ result, err := GetBlock(TEST_HASH)
+ if err != NotFoundError {
+ t.Errorf("Expected NotFoundError, got %v", result)
+ }
+}
+
+// TestGetBlockCorrupt
+// GetBlock must return an error when a corrupted block is requested
+// (the contents of the file do not checksum to its hash).
+//
+func TestGetBlockCorrupt(t *testing.T) {
+ defer teardown()
+
+ // Create two test Keep volumes and store a block in each of them,
+ // but the hash of the block does not match the filename.
+ KeepVolumes = setup(t, 2)
+ for _, vol := range KeepVolumes {
+ store(t, vol, TEST_HASH, BAD_BLOCK)
+ }
+
+ // Check that GetBlock returns failure.
+ result, err := GetBlock(TEST_HASH)
+ if err != CorruptError {
+ t.Errorf("Expected CorruptError, got %v", result)
+ }
+}
+
+// ========================================
+// PutBlock tests
+// ========================================
+
+// TestPutBlockOK
+// PutBlock can perform a simple block write and returns success.
+//
+func TestPutBlockOK(t *testing.T) {
+ defer teardown()
+
+ // Create two test Keep volumes.
+ KeepVolumes = setup(t, 2)
+
+ // Check that PutBlock stores the data as expected.
+ if err := PutBlock(TEST_BLOCK, TEST_HASH); err != nil {
+ t.Fatalf("PutBlock: %v", err)
+ }
+
+ result, err := GetBlock(TEST_HASH)
+ if err != nil {
+ t.Fatalf("GetBlock returned error: %v", err)
+ }
+ if string(result) != string(TEST_BLOCK) {
+ t.Error("PutBlock/GetBlock mismatch")
+ t.Fatalf("PutBlock stored '%s', GetBlock retrieved '%s'",
+ string(TEST_BLOCK), string(result))
+ }
+}
+
+// TestPutBlockOneVol
+// PutBlock still returns success even when only one of the known
+// volumes is online.
+//
+func TestPutBlockOneVol(t *testing.T) {
+ defer teardown()
+
+ // Create two test Keep volumes, but cripple one of them.
+ KeepVolumes = setup(t, 2)
+ os.Chmod(KeepVolumes[0], 000)
+
+ // Check that PutBlock stores the data as expected.
+ if err := PutBlock(TEST_BLOCK, TEST_HASH); err != nil {
+ t.Fatalf("PutBlock: %v", err)
+ }
+
+ result, err := GetBlock(TEST_HASH)
+ if err != nil {
+ t.Fatalf("GetBlock: %v", err)
+ }
+ if string(result) != string(TEST_BLOCK) {
+ t.Error("PutBlock/GetBlock mismatch")
+ t.Fatalf("PutBlock stored '%s', GetBlock retrieved '%s'",
+ string(TEST_BLOCK), string(result))
+ }
+}
+
+// TestPutBlockMD5Fail
+// Check that PutBlock returns an error if passed a block and hash that
+// do not match.
+//
+func TestPutBlockMD5Fail(t *testing.T) {
+ defer teardown()
+
+ // Create two test Keep volumes.
+ KeepVolumes = setup(t, 2)
+
+ // Check that PutBlock returns the expected error when the hash does
+ // not match the block.
+ if err := PutBlock(BAD_BLOCK, TEST_HASH); err != MD5Error {
+ t.Error("Expected MD5Error, got %v", err)
+ }
+
+ // Confirm that GetBlock fails to return anything.
+ if result, err := GetBlock(TEST_HASH); err != NotFoundError {
+ t.Errorf("GetBlock succeeded after a corrupt block store (result = %s, err = %v)",
+ string(result), err)
+ }
+}
+
+// TestPutBlockCorrupt
+// PutBlock should overwrite corrupt blocks on disk when given
+// a PUT request with a good block.
+//
+func TestPutBlockCorrupt(t *testing.T) {
+ defer teardown()
+
+ // Create two test Keep volumes.
+ KeepVolumes = setup(t, 2)
+
+ // Store a corrupted block under TEST_HASH.
+ store(t, KeepVolumes[0], TEST_HASH, BAD_BLOCK)
+ if err := PutBlock(TEST_BLOCK, TEST_HASH); err != nil {
+ t.Errorf("PutBlock: %v", err)
+ }
+
+ // The block on disk should now match TEST_BLOCK.
+ if block, err := GetBlock(TEST_HASH); err != nil {
+ t.Errorf("GetBlock: %v", err)
+ } else if bytes.Compare(block, TEST_BLOCK) != 0 {
+ t.Errorf("GetBlock returned: '%s'", string(block))
+ }
+}
+
+// PutBlockCollision
+// PutBlock returns a 400 Collision error when attempting to
+// store a block that collides with another block on disk.
+//
+func TestPutBlockCollision(t *testing.T) {
+ defer teardown()
+
+ // These blocks both hash to the MD5 digest cee9a457e790cf20d4bdaa6d69f01e41.
+ var b1 = []byte("\x0e0eaU\x9a\xa7\x87\xd0\x0b\xc6\xf7\x0b\xbd\xfe4\x04\xcf\x03e\x9epO\x854\xc0\x0f\xfbe\x9cL\x87@\xcc\x94/\xeb-\xa1\x15\xa3\xf4\x15\\\xbb\x86\x07Is\x86em}\x1f4\xa4 Y\xd7\x8fZ\x8d\xd1\xef")
+ var b2 = []byte("\x0e0eaU\x9a\xa7\x87\xd0\x0b\xc6\xf7\x0b\xbd\xfe4\x04\xcf\x03e\x9etO\x854\xc0\x0f\xfbe\x9cL\x87@\xcc\x94/\xeb-\xa1\x15\xa3\xf4\x15\xdc\xbb\x86\x07Is\x86em}\x1f4\xa4 Y\xd7\x8fZ\x8d\xd1\xef")
+ var locator = "cee9a457e790cf20d4bdaa6d69f01e41"
+
+ // Prepare two test Keep volumes. Store one block,
+ // then attempt to store the other.
+ KeepVolumes = setup(t, 2)
+ store(t, KeepVolumes[1], locator, b1)
+
+ if err := PutBlock(b2, locator); err == nil {
+ t.Error("PutBlock did not report a collision")
+ } else if err != CollisionError {
+ t.Errorf("PutBlock returned %v", err)
+ }
+}
+
+// ========================================
+// FindKeepVolumes tests.
+// ========================================
+
+// TestFindKeepVolumes
+// Confirms that FindKeepVolumes finds tmpfs volumes with "/keep"
+// directories at the top level.
+//
+func TestFindKeepVolumes(t *testing.T) {
+ defer teardown()
+
+ // Initialize two keep volumes.
+ var tempVols []string = setup(t, 2)
+
+ // Set up a bogus PROC_MOUNTS file.
+ if f, err := ioutil.TempFile("", "keeptest"); err == nil {
+ for _, vol := range tempVols {
+ fmt.Fprintf(f, "tmpfs %s tmpfs opts\n", path.Dir(vol))
+ }
+ f.Close()
+ PROC_MOUNTS = f.Name()
+
+ // Check that FindKeepVolumes finds the temp volumes.
+ resultVols := FindKeepVolumes()
+ if len(tempVols) != len(resultVols) {
+ t.Fatalf("set up %d volumes, FindKeepVolumes found %d\n",
+ len(tempVols), len(resultVols))
+ }
+ for i := range tempVols {
+ if tempVols[i] != resultVols[i] {
+ t.Errorf("FindKeepVolumes returned %s, expected %s\n",
+ resultVols[i], tempVols[i])
+ }
+ }
+
+ os.Remove(f.Name())
+ }
+}
+
+// TestFindKeepVolumesFail
+// When no Keep volumes are present, FindKeepVolumes returns an empty slice.
+//
+func TestFindKeepVolumesFail(t *testing.T) {
+ defer teardown()
+
+ // Set up a bogus PROC_MOUNTS file with no Keep vols.
+ if f, err := ioutil.TempFile("", "keeptest"); err == nil {
+ fmt.Fprintln(f, "rootfs / rootfs opts 0 0")
+ fmt.Fprintln(f, "sysfs /sys sysfs opts 0 0")
+ fmt.Fprintln(f, "proc /proc proc opts 0 0")
+ fmt.Fprintln(f, "udev /dev devtmpfs opts 0 0")
+ fmt.Fprintln(f, "devpts /dev/pts devpts opts 0 0")
+ f.Close()
+ PROC_MOUNTS = f.Name()
+
+ // Check that FindKeepVolumes returns an empty array.
+ resultVols := FindKeepVolumes()
+ if len(resultVols) != 0 {
+ t.Fatalf("FindKeepVolumes returned %v", resultVols)
+ }
+
+ os.Remove(PROC_MOUNTS)
+ }
+}
+
+// ========================================
+// Helper functions for unit tests.
+// ========================================
+
+// setup
+// Create KeepVolumes for testing.
+// Returns a slice of pathnames to temporary Keep volumes.
+//
+func setup(t *testing.T, num_volumes int) []string {
+ vols := make([]string, num_volumes)
+ for i := range vols {
+ if dir, err := ioutil.TempDir(os.TempDir(), "keeptest"); err == nil {
+ vols[i] = dir + "/keep"
+ os.Mkdir(vols[i], 0755)
+ } else {
+ t.Fatal(err)
+ }
+ }
+ return vols
+}
+
+// teardown
+// Cleanup to perform after each test.
+//
+func teardown() {
+ for _, vol := range KeepVolumes {
+ os.RemoveAll(path.Dir(vol))
+ }
+ KeepVolumes = nil
+}
+
+// store
+// Low-level code to write Keep blocks directly to disk for testing.
+//
+func store(t *testing.T, keepdir string, filename string, block []byte) {
+ blockdir := fmt.Sprintf("%s/%s", keepdir, filename[:3])
+ if err := os.MkdirAll(blockdir, 0755); err != nil {
+ t.Fatal(err)
+ }
+
+ blockpath := fmt.Sprintf("%s/%s", blockdir, filename)
+ if f, err := os.Create(blockpath); err == nil {
+ f.Write(block)
+ f.Close()
+ } else {
+ t.Fatal(err)
+ }
+}
projects / arvados.git / commitdiff
