6649: Merge branch 'master' into 6649-document-webshell

Arvados-DCO-1.1-Signed-off-by: Ward Vandewege <ward@curii.com>

diff --git a/apps/workbench/Gemfile.lock b/apps/workbench/Gemfile.lock
index 2420fee24d07e056d3ee4b7047f43f87dd1b5d6d..cb4e7ab9e334cb8fdb0ae72c20ee841f4fed02b2 100644 (file)
--- a/apps/workbench/Gemfile.lock
+++ b/apps/workbench/Gemfile.lock
@@ -214,7 +214,7 @@ GEM
       multi_json (~> 1.0)
       websocket-driver (>= 0.2.0)
     public_suffix (4.0.3)
-    rack (2.2.2)
+    rack (2.2.3)
     rack-mini-profiler (1.0.2)
       rack (>= 1.2.0)
     rack-test (0.6.3)

diff --git a/build/package-testing/test-package-python3-python-arvados-fuse.sh b/build/package-testing/test-package-python3-python-arvados-fuse.sh
new file mode 120000 (symlink)
index 0000000..3b9232c
--- /dev/null
+++ b/build/package-testing/test-package-python3-python-arvados-fuse.sh
@@ -0,0 +1 @@
+test-package-python27-python-arvados-fuse.sh
\ No newline at end of file

diff --git a/build/run-build-packages.sh b/build/run-build-packages.sh
index 3ba1dcc05e8776fc57a205e2deb79a0224a8e370..862b93e6e4667ad561c478e13b0f6f53ee2d012f 100755 (executable)
--- a/build/run-build-packages.sh
+++ b/build/run-build-packages.sh
@@ -334,6 +334,9 @@ fpm_build_virtualenv "libpam-arvados" "sdk/pam"
 # The FUSE driver
 fpm_build_virtualenv "arvados-fuse" "services/fuse"
 
+# The FUSE driver - Python3 package
+fpm_build_virtualenv "arvados-fuse" "services/fuse" "python3"
+
 # The node manager
 fpm_build_virtualenv "arvados-node-manager" "services/nodemanager"
 

diff --git a/build/run-library.sh b/build/run-library.sh
index b75b6cca78bae2f6e11fa309c08b921f874f2ae5..f8e5129daeb0ce63aba2230c04214ca252bba476 100755 (executable)
--- a/build/run-library.sh
+++ b/build/run-library.sh
@@ -574,7 +574,7 @@ fpm_build_virtualenv () {
   cd build/usr/share/$python/dist/$PYTHON_PKG/
 
   # Replace the shebang lines in all python scripts, and handle the activate
-  # scripts too This is a functional replacement of the 237 line
+  # scripts too. This is a functional replacement of the 237 line
   # virtualenv_tools.py script that doesn't work in python3 without serious
   # patching, minus the parts we don't need (modifying pyc files, etc).
   for binfile in `ls bin/`; do
@@ -632,7 +632,7 @@ fpm_build_virtualenv () {
     COMMAND_ARR+=('--rpm-auto-add-directories')
   fi
 
-  if [[ "$PKG" == "arvados-python-client" ]]; then
+  if [[ "$PKG" == "arvados-python-client" ]] || [[ "$PKG" == "arvados-fuse" ]]; then
     if [[ "$python" == "python2.7" ]]; then
       COMMAND_ARR+=('--conflicts' "$PYTHON3_PKG_PREFIX-$PKG")
     else

diff --git a/doc/install/install-shell-server.html.textile.liquid b/doc/install/install-shell-server.html.textile.liquid
index 44b3834ab84ec8df76d4810c1ee76dbaf7fa0845..5ac5e9e6b870a2753287b2b8a59e50c6686d80df 100644 (file)
--- a/doc/install/install-shell-server.html.textile.liquid
+++ b/doc/install/install-shell-server.html.textile.liquid
@@ -69,7 +69,7 @@ As an Arvados admin user (such as the system root user), create a "scoped token"
 
 <notextile>
 <pre>
-<code>apiserver:~$ <span class="userinput">arv api_client_authorization create --api-client-authorization '{"scopes":["GET /arvados/v1/virtual_machines/<b>zzzzz-2x53u-zzzzzzzzzzzzzzz</b>/logins"]}'
+<code>apiserver:~$ <span class="userinput">arv api_client_authorization create --api-client-authorization '{"scopes":["GET /arvados/v1/virtual_machines/<b>zzzzz-2x53u-zzzzzzzzzzzzzzz</b>/logins"]}'</span>
 {
  ...
  "api_token":"zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz",

diff --git a/docker/jobs/Dockerfile b/docker/jobs/Dockerfile
index 876ac4f9f49cea14c42f54f1ebe37423b4251cd2..15993c4bc322619e125ddb5411a79a2d0f4348f0 100644 (file)
--- a/docker/jobs/Dockerfile
+++ b/docker/jobs/Dockerfile
@@ -3,8 +3,8 @@
 # SPDX-License-Identifier: Apache-2.0
 
 # Based on Debian Stretch
-FROM debian:stretch-slim
-MAINTAINER Peter Amstutz <peter.amstutz@curii.com>
+FROM debian:buster-slim
+MAINTAINER Arvados Package Maintainers <packaging@arvados.org>
 
 ENV DEBIAN_FRONTEND noninteractive
 

diff --git a/docker/jobs/apt.arvados.org-dev.list b/docker/jobs/apt.arvados.org-dev.list
index 468000ed29b9244460e28f0b5abe5f5efd13f133..4de87397bca754a57e384c3155d88b82a30983fc 100644 (file)
--- a/docker/jobs/apt.arvados.org-dev.list
+++ b/docker/jobs/apt.arvados.org-dev.list
@@ -1,2 +1,2 @@
 # apt.arvados.org
-deb http://apt.arvados.org/ stretch-dev main
+deb http://apt.arvados.org/ buster-dev main

diff --git a/docker/jobs/apt.arvados.org-stable.list b/docker/jobs/apt.arvados.org-stable.list
index afbc51effe84979f49f5d1c9584bf951c2408922..7882afd01c96235b1fde32767d56a68aeada8d03 100644 (file)
--- a/docker/jobs/apt.arvados.org-stable.list
+++ b/docker/jobs/apt.arvados.org-stable.list
@@ -1,2 +1,2 @@
 # apt.arvados.org
-deb http://apt.arvados.org/ stretch main
+deb http://apt.arvados.org/ buster main

diff --git a/docker/jobs/apt.arvados.org-testing.list b/docker/jobs/apt.arvados.org-testing.list
index c8ea91d070a572365006e849015d48006d060a22..3bb599087eaf513bb5c3f6dc2e32d54108d3db53 100644 (file)
--- a/docker/jobs/apt.arvados.org-testing.list
+++ b/docker/jobs/apt.arvados.org-testing.list
@@ -1,2 +1,2 @@
 # apt.arvados.org
-deb http://apt.arvados.org/ stretch-testing main
+deb http://apt.arvados.org/ buster-testing main

diff --git a/sdk/cwl/arvados_cwl/runner.py b/sdk/cwl/arvados_cwl/runner.py
index 7bb66a158e50646f1fc984df7ea30ccd31528bf3..a47b34acc0b31d304e56f98d89a6d277e598c9ea 100644 (file)
--- a/sdk/cwl/arvados_cwl/runner.py
+++ b/sdk/cwl/arvados_cwl/runner.py
@@ -169,21 +169,47 @@ def set_secondary(fsaccess, builder, inputschema, secondaryspec, primary, discov
         #
         # Found a file, check for secondaryFiles
         #
-        primary["secondaryFiles"] = []
+        specs = []
+        primary["secondaryFiles"] = secondaryspec
         for i, sf in enumerate(aslist(secondaryspec)):
             pattern = builder.do_eval(sf["pattern"], context=primary)
             if pattern is None:
                 continue
+            if isinstance(pattern, list):
+                specs.extend(pattern)
+            elif isinstance(pattern, dict):
+                specs.append(pattern)
+            elif isinstance(pattern, str):
+                specs.append({"pattern": pattern})
+            else:
+                raise SourceLine(primary["secondaryFiles"], i, validate.ValidationException).makeError(
+                    "Expression must return list, object, string or null")
+
+        found = []
+        for i, sf in enumerate(specs):
+            if isinstance(sf, dict):
+                if sf.get("class") == "File":
+                    pattern = sf["basename"]
+                else:
+                    pattern = sf["pattern"]
+                    required = sf.get("required")
+            elif isinstance(sf, str):
+                pattern = sf
+                required = True
+            else:
+                raise SourceLine(primary["secondaryFiles"], i, validate.ValidationException).makeError(
+                    "Expression must return list, object, string or null")
+
             sfpath = substitute(primary["location"], pattern)
-            required = builder.do_eval(sf.get("required"), context=primary)
+            required = builder.do_eval(required, context=primary)
 
             if fsaccess.exists(sfpath):
-                primary["secondaryFiles"].append({"location": sfpath, "class": "File"})
+                found.append({"location": sfpath, "class": "File"})
             elif required:
                 raise SourceLine(primary["secondaryFiles"], i, validate.ValidationException).makeError(
                     "Required secondary file '%s' does not exist" % sfpath)
 
-        primary["secondaryFiles"] = cmap(primary["secondaryFiles"])
+        primary["secondaryFiles"] = cmap(found)
         if discovered is not None:
             discovered[primary["location"]] = primary["secondaryFiles"]
     elif inputschema["type"] not in primitive_types_set:
@@ -434,9 +460,11 @@ def packed_workflow(arvrunner, tool, merged_map):
     def visit(v, cur_id):
         if isinstance(v, dict):
             if v.get("class") in ("CommandLineTool", "Workflow"):
-                if "id" not in v:
-                    raise SourceLine(v, None, Exception).makeError("Embedded process object is missing required 'id' field")
-                cur_id = rewrite_to_orig.get(v["id"], v["id"])
+                if "id" in v:
+                    cur_id = rewrite_to_orig.get(v["id"], v["id"])
+            if "path" in v and "location" not in v:
+                v["location"] = v["path"]
+                del v["path"]
             if "location" in v and not v["location"].startswith("keep:"):
                 v["location"] = merged_map[cur_id].resolved[v["location"]]
             if "location" in v and v["location"] in merged_map[cur_id].secondaryFiles:

diff --git a/services/api/Gemfile.lock b/services/api/Gemfile.lock
index c8a1a27b79c1e939438dcb8ed603c206a299c409..127a09ee2db71a00bc7c05ee5e2e651ea379a33d 100644 (file)
--- a/services/api/Gemfile.lock
+++ b/services/api/Gemfile.lock
@@ -180,7 +180,7 @@ GEM
     pg (1.1.4)
     power_assert (1.1.4)
     public_suffix (4.0.3)
-    rack (2.2.2)
+    rack (2.2.3)
     rack-test (0.6.3)
       rack (>= 1.0)
     rails (5.0.7.2)

diff --git a/services/api/lib/fix_roles_projects.rb b/services/api/lib/fix_roles_projects.rb
index 5dd127b3e230246962b179f2431d9ad49a95483a..5bb013c9add7a1f241d4779768cef462ac9956b2 100644 (file)
--- a/services/api/lib/fix_roles_projects.rb
+++ b/services/api/lib/fix_roles_projects.rb
@@ -2,6 +2,8 @@
 #
 # SPDX-License-Identifier: AGPL-3.0
 
+require 'update_permissions'
+
 include CurrentApiClient
 
 def fix_roles_projects
@@ -20,10 +22,11 @@ def fix_roles_projects
 
           if old_owner != system_user_uuid
             # 2) Ownership of a role becomes a can_manage link
-            Link.create!(link_class: 'permission',
+            Link.new(link_class: 'permission',
                          name: 'can_manage',
                          tail_uuid: old_owner,
-                         head_uuid: g.uuid)
+                         head_uuid: g.uuid).
+              save!(validate: false)
           end
         end
 
@@ -37,20 +40,22 @@ def fix_roles_projects
           # 3) If a role owns anything, give it to system user and it
           # becomes a can_manage link
           klass.joins("join groups on groups.uuid=#{klass.table_name}.owner_uuid and groups.group_class='role'").each do |owned|
-            Link.create!(link_class: 'permission',
-                         name: 'can_manage',
-                         tail_uuid: owned.owner_uuid,
-                         head_uuid: owned.uuid)
+            Link.new(link_class: 'permission',
+                     name: 'can_manage',
+                     tail_uuid: owned.owner_uuid,
+                     head_uuid: owned.uuid).
+              save!(validate: false)
             owned.owner_uuid = system_user_uuid
             owned.save_with_unique_name!
           end
         end
 
         Group.joins("join groups as g2 on g2.uuid=groups.owner_uuid and g2.group_class='role'").each do |owned|
-          Link.create!(link_class: 'permission',
+          Link.new(link_class: 'permission',
                        name: 'can_manage',
                        tail_uuid: owned.owner_uuid,
-                       head_uuid: owned.uuid)
+                       head_uuid: owned.uuid).
+            save!(validate: false)
           owned.owner_uuid = system_user_uuid
           owned.save_with_unique_name!
         end

diff --git a/services/api/test/unit/group_test.rb b/services/api/test/unit/group_test.rb
index 3d1fda927f0554ce7955a4e73d7e0e8921f7d8a5..30fddfa5b8be8f89c2f43651c1c316a6e59253fe 100644 (file)
--- a/services/api/test/unit/group_test.rb
+++ b/services/api/test/unit/group_test.rb
@@ -6,6 +6,7 @@ require 'test_helper'
 require 'fix_roles_projects'
 
 class GroupTest < ActiveSupport::TestCase
+  include DbCurrentTime
 
   test "cannot set owner_uuid to object with existing ownership cycle" do
     set_user_from_auth :active_trustedclient
@@ -317,6 +318,14 @@ insert into groups (uuid, owner_uuid, name, group_class, created_at, updated_at)
     g6 = insert_group Group.generate_uuid, system_user_uuid, 'name collision', 'role'
     g7 = insert_group Group.generate_uuid, users(:active).uuid, 'name collision', 'role'
 
+    g8 = insert_group Group.generate_uuid, users(:active).uuid, 'trashed with no class', nil
+    g8obj = Group.find_by_uuid(g8)
+    g8obj.trash_at = db_current_time
+    g8obj.delete_at = db_current_time
+    act_as_system_user do
+      g8obj.save!(validate: false)
+    end
+
     refresh_permissions
 
     act_as_system_user do
@@ -328,6 +337,7 @@ update links set tail_uuid='#{g5}' where uuid='#{l1.uuid}'
     end
 
     assert_equal nil, Group.find_by_uuid(g1).group_class
+    assert_equal nil, Group.find_by_uuid(g8).group_class
     assert_equal users(:active).uuid, Group.find_by_uuid(g2).owner_uuid
     assert_equal g3, Group.find_by_uuid(g4).owner_uuid
     assert !Link.where(tail_uuid: users(:active).uuid, head_uuid: g2, link_class: "permission", name: "can_manage").any?
@@ -337,6 +347,7 @@ update links set tail_uuid='#{g5}' where uuid='#{l1.uuid}'
     fix_roles_projects
 
     assert_equal 'role', Group.find_by_uuid(g1).group_class
+    assert_equal 'role', Group.find_by_uuid(g8).group_class
     assert_equal system_user_uuid, Group.find_by_uuid(g2).owner_uuid
     assert_equal system_user_uuid, Group.find_by_uuid(g4).owner_uuid
     assert Link.where(tail_uuid: users(:active).uuid, head_uuid: g2, link_class: "permission", name: "can_manage").any?

diff --git a/services/fuse/fpm-info.sh b/services/fuse/fpm-info.sh
index fd94ef7afa3340edea84a486e4abd03810fe1b8d..f789abe69270c024e73a5294666bc06169b45026 100644 (file)
--- a/services/fuse/fpm-info.sh
+++ b/services/fuse/fpm-info.sh
@@ -9,6 +9,6 @@ case "$TARGET" in
         fpm_depends+=(fuse-libs)
         ;;
     debian* | ubuntu*)
-        fpm_depends+=(libcurl3-gnutls libpython2.7)
+        fpm_depends+=(libcurl3-gnutls)
         ;;
 esac