From: Brett Smith Date: Fri, 10 May 2024 17:15:28 +0000 (-0400) Subject: Merge branch '21207-pytest' X-Git-Url: https://git.arvados.org/arvados.git/commitdiff_plain/6a1a9849137351b7ddbfe52011174ea3182f4b30?hp=c28159e9eda5bdd027792a20fa8d0ad04efb74f7 Merge branch '21207-pytest' Closes #21207, #21733. Arvados-DCO-1.1-Signed-off-by: Brett Smith --- diff --git a/.licenseignore b/.licenseignore index 1e1c12a53a..926aff75a4 100644 --- a/.licenseignore +++ b/.licenseignore @@ -43,6 +43,7 @@ docker/jobs/1078ECD7.key *.min.js *.png */proc_stat +*/pytest.ini */README */robots.txt */runit-docker/* diff --git a/build/run-tests.sh b/build/run-tests.sh index 3ab571ff7f..0eb421454e 100755 --- a/build/run-tests.sh +++ b/build/run-tests.sh @@ -38,7 +38,7 @@ WORKSPACE=path Arvados source tree to test. CONFIGSRC=path Dir with config.yml file containing PostgreSQL section for use by tests. services/api_test="TEST=test/functional/arvados/v1/collections_controller_test.rb" Restrict apiserver tests to the given file -sdk/python_test="--test-suite tests.test_keep_locator" +sdk/python_test="tests/test_api.py::ArvadosApiTest" Restrict Python SDK tests to the given class lib/dispatchcloud_test="-check.vv" Show all log messages, even when tests pass (also works @@ -566,13 +566,14 @@ install_env() { setup_virtualenv "$VENV3DIR" . "$VENV3DIR/bin/activate" - # wheel modernizes the venv (as of early 2024) and makes it more closely - # match our package build environment. + # parameterized and pytest are direct dependencies of Python tests. # PyYAML is a test requirement used by run_test_server.py and needed for # other, non-Python tests. # pdoc is needed to build PySDK documentation. + # wheel modernizes the venv (as of early 2024) and makes it more closely + # match our package build environment. # We run `setup.py build` first to generate _version.py. - pip install PyYAML pdoc wheel \ + pip install parameterized pytest PyYAML pdoc wheel \ && env -C "$WORKSPACE/sdk/python" python3 setup.py build \ && pip install "$WORKSPACE/sdk/python" \ || fatal "installing Python SDK and related dependencies failed" @@ -701,7 +702,7 @@ do_test_once() { elif [[ "$2" == "pip" ]] then tries=0 - cd "$WORKSPACE/$1" && while : + cd "$WORKSPACE/$1" && pip install . && while : do tries=$((${tries}+1)) # $3 can name a path directory for us to use, including trailing @@ -709,11 +710,13 @@ do_test_once() { if [[ -e "${3}activate" ]]; then . "${3}activate" fi - python setup.py ${short:+--short-tests-only} test ${testargs[$1]} + python3 -m pytest ${testargs[$1]} result=$? - if [[ ${tries} < 3 && ${result} == 137 ]] + # pytest uses exit code 2 to mean "test collection failed." + # See discussion in FUSE's IntegrationTest and MountTestBase. + if [[ ${tries} < 3 && ${result} == 2 ]] then - printf '\n*****\n%s tests killed -- retrying\n*****\n\n' "$1" + printf '\n*****\n%s tests exited with code 2 -- retrying\n*****\n\n' "$1" continue else break diff --git a/sdk/cwl/arvados_version.py b/sdk/cwl/arvados_version.py index 794b6afe42..ca20de4c58 100644 --- a/sdk/cwl/arvados_version.py +++ b/sdk/cwl/arvados_version.py @@ -72,14 +72,6 @@ if REPO_PATH is None: if (SETUP_DIR / mod_name).is_dir() ) -def short_tests_only(arglist=sys.argv): - try: - arglist.remove('--short-tests-only') - except ValueError: - return False - else: - return True - def git_log_output(path, *args): return subprocess.check_output( ['git', '-C', str(REPO_PATH), diff --git a/sdk/cwl/pytest.ini b/sdk/cwl/pytest.ini new file mode 120000 index 0000000000..05a82dbfef --- /dev/null +++ b/sdk/cwl/pytest.ini @@ -0,0 +1 @@ +../../sdk/python/pytest.ini \ No newline at end of file diff --git a/sdk/cwl/tests/test_copy_deps.py b/sdk/cwl/tests/test_copy_deps.py index 28a5915b11..8ad735fddc 100644 --- a/sdk/cwl/tests/test_copy_deps.py +++ b/sdk/cwl/tests/test_copy_deps.py @@ -73,7 +73,7 @@ def check_contents(group, wf_uuid): raise Exception("Couldn't find collection containing expected "+expect_file) -def test_create(): +def check_create(): group = api.groups().create(body={"group": {"name": "test-19070-project-1", "group_class": "project"}}, ensure_unique_name=True).execute() try: contents = api.groups().contents(uuid=group["uuid"]).execute() @@ -90,7 +90,7 @@ def test_create(): api.groups().delete(uuid=group["uuid"]).execute() -def test_update(): +def check_update(): group = api.groups().create(body={"group": {"name": "test-19070-project-2", "group_class": "project"}}, ensure_unique_name=True).execute() try: contents = api.groups().contents(uuid=group["uuid"]).execute() @@ -132,7 +132,7 @@ def test_update(): api.groups().delete(uuid=group["uuid"]).execute() -def test_execute(): +def check_execute(): group = api.groups().create(body={"group": {"name": "test-19070-project-3", "group_class": "project"}}, ensure_unique_name=True).execute() try: contents = api.groups().contents(uuid=group["uuid"]).execute() @@ -193,6 +193,6 @@ def test_execute(): api.groups().delete(uuid=group["uuid"]).execute() if __name__ == '__main__': - test_create() - test_update() - test_execute() + check_create() + check_update() + check_execute() diff --git a/sdk/cwl/tests/test_set_output_prop.py b/sdk/cwl/tests/test_set_output_prop.py index 3219eac989..0e829eeb92 100644 --- a/sdk/cwl/tests/test_set_output_prop.py +++ b/sdk/cwl/tests/test_set_output_prop.py @@ -7,7 +7,7 @@ import subprocess api = arvados.api() -def test_execute(): +def check_execute(): group = api.groups().create(body={"group": {"name": "test-17004-project", "group_class": "project"}}, ensure_unique_name=True).execute() try: contents = api.groups().contents(uuid=group["uuid"]).execute() @@ -34,4 +34,4 @@ def test_execute(): api.groups().delete(uuid=group["uuid"]).execute() if __name__ == '__main__': - test_execute() + check_execute() diff --git a/sdk/python/arvados_version.py b/sdk/python/arvados_version.py index 794b6afe42..ca20de4c58 100644 --- a/sdk/python/arvados_version.py +++ b/sdk/python/arvados_version.py @@ -72,14 +72,6 @@ if REPO_PATH is None: if (SETUP_DIR / mod_name).is_dir() ) -def short_tests_only(arglist=sys.argv): - try: - arglist.remove('--short-tests-only') - except ValueError: - return False - else: - return True - def git_log_output(path, *args): return subprocess.check_output( ['git', '-C', str(REPO_PATH), diff --git a/sdk/python/pytest.ini b/sdk/python/pytest.ini new file mode 100644 index 0000000000..9b1bbfd50a --- /dev/null +++ b/sdk/python/pytest.ini @@ -0,0 +1,8 @@ +#!/usr/bin/env python3 +# Copyright (C) The Arvados Authors. All rights reserved. +# +# SPDX-License-Identifier: Apache-2.0 + +[pytest] +testpaths = + tests diff --git a/sdk/python/setup.py b/sdk/python/setup.py index 257968d89e..dc10269060 100644 --- a/sdk/python/setup.py +++ b/sdk/python/setup.py @@ -13,7 +13,6 @@ from setuptools.command import build_py import arvados_version version = arvados_version.get_version() -short_tests_only = arvados_version.short_tests_only() README = os.path.join(arvados_version.SETUP_DIR, 'README.rst') class BuildPython(build_py.build_py): diff --git a/sdk/python/tests/slow_test.py b/sdk/python/tests/slow_test.py deleted file mode 100644 index ae46f4e1bf..0000000000 --- a/sdk/python/tests/slow_test.py +++ /dev/null @@ -1,11 +0,0 @@ -# Copyright (C) The Arvados Authors. All rights reserved. -# -# SPDX-License-Identifier: Apache-2.0 - -import __main__ -import os -import unittest - -slow_test = lambda _: unittest.skipIf( - __main__.short_tests_only, - "running --short tests only") diff --git a/sdk/python/tests/test_arv_put.py b/sdk/python/tests/test_arv_put.py index e3f7c9d446..e8a3e65bdc 100644 --- a/sdk/python/tests/test_arv_put.py +++ b/sdk/python/tests/test_arv_put.py @@ -6,6 +6,7 @@ import apiclient import ciso8601 +import copy import datetime import json import logging @@ -568,7 +569,7 @@ class ArvPutUploadJobTest(run_test_server.TestCaseWithServers, class CachedManifestValidationTest(ArvadosBaseTestCase): class MockedPut(arv_put.ArvPutUploadJob): def __init__(self, cached_manifest=None): - self._state = arv_put.ArvPutUploadJob.EMPTY_STATE + self._state = copy.deepcopy(arv_put.ArvPutUploadJob.EMPTY_STATE) self._state['manifest'] = cached_manifest self._api_client = mock.MagicMock() self.logger = mock.MagicMock() diff --git a/services/dockercleaner/arvados_version.py b/services/dockercleaner/arvados_version.py index 794b6afe42..ca20de4c58 100644 --- a/services/dockercleaner/arvados_version.py +++ b/services/dockercleaner/arvados_version.py @@ -72,14 +72,6 @@ if REPO_PATH is None: if (SETUP_DIR / mod_name).is_dir() ) -def short_tests_only(arglist=sys.argv): - try: - arglist.remove('--short-tests-only') - except ValueError: - return False - else: - return True - def git_log_output(path, *args): return subprocess.check_output( ['git', '-C', str(REPO_PATH), diff --git a/services/dockercleaner/pytest.ini b/services/dockercleaner/pytest.ini new file mode 120000 index 0000000000..05a82dbfef --- /dev/null +++ b/services/dockercleaner/pytest.ini @@ -0,0 +1 @@ +../../sdk/python/pytest.ini \ No newline at end of file diff --git a/services/dockercleaner/setup.py b/services/dockercleaner/setup.py index b20b776ebf..cb0fc8d88a 100644 --- a/services/dockercleaner/setup.py +++ b/services/dockercleaner/setup.py @@ -11,7 +11,6 @@ from setuptools import setup, find_packages import arvados_version version = arvados_version.get_version() -short_tests_only = arvados_version.short_tests_only() README = os.path.join(arvados_version.SETUP_DIR, 'README.rst') setup(name="arvados-docker-cleaner", diff --git a/services/fuse/arvados_version.py b/services/fuse/arvados_version.py index 794b6afe42..ca20de4c58 100644 --- a/services/fuse/arvados_version.py +++ b/services/fuse/arvados_version.py @@ -72,14 +72,6 @@ if REPO_PATH is None: if (SETUP_DIR / mod_name).is_dir() ) -def short_tests_only(arglist=sys.argv): - try: - arglist.remove('--short-tests-only') - except ValueError: - return False - else: - return True - def git_log_output(path, *args): return subprocess.check_output( ['git', '-C', str(REPO_PATH), diff --git a/services/fuse/pytest.ini b/services/fuse/pytest.ini new file mode 120000 index 0000000000..05a82dbfef --- /dev/null +++ b/services/fuse/pytest.ini @@ -0,0 +1 @@ +../../sdk/python/pytest.ini \ No newline at end of file diff --git a/services/fuse/setup.py b/services/fuse/setup.py index 002afc22c9..f9b0fcf91d 100644 --- a/services/fuse/setup.py +++ b/services/fuse/setup.py @@ -11,7 +11,6 @@ from setuptools import setup, find_packages import arvados_version version = arvados_version.get_version() -short_tests_only = arvados_version.short_tests_only() README = os.path.join(arvados_version.SETUP_DIR, 'README.rst') setup(name='arvados_fuse', diff --git a/services/fuse/tests/integration_test.py b/services/fuse/tests/integration_test.py index ae1b58ad47..24ac7baf04 100644 --- a/services/fuse/tests/integration_test.py +++ b/services/fuse/tests/integration_test.py @@ -16,6 +16,8 @@ import sys import tempfile import unittest +import pytest + from . import run_test_server @atexit.register @@ -91,9 +93,15 @@ class IntegrationTest(unittest.TestCase): return func(self, *args, **kwargs) finally: if self.mount and self.mount.llfuse_thread.is_alive(): - logging.warning("IntegrationTest.mount:" - " llfuse thread still alive after umount" - " -- killing test suite to avoid deadlock") - os.kill(os.getpid(), signal.SIGKILL) + # pytest uses exit status 2 when test collection failed. + # A UnitTest failing in setup/teardown counts as a + # collection failure, so pytest will exit with status 2 + # no matter what status you specify here. run-tests.sh + # looks for this status, so specify 2 just to keep + # everything as consistent as possible. + # TODO: If we refactor these tests so they're not built + # on unittest, consider using a dedicated, non-pytest + # exit code like TEMPFAIL. + pytest.exit("llfuse thread outlived test - aborting test suite to avoid deadlock", 2) return wrapper return decorator diff --git a/services/fuse/tests/mount_test_base.py b/services/fuse/tests/mount_test_base.py index 6e49390524..9768aeb74d 100644 --- a/services/fuse/tests/mount_test_base.py +++ b/services/fuse/tests/mount_test_base.py @@ -19,6 +19,8 @@ import threading import time import unittest +import pytest + from . import run_test_server from .integration_test import workerPool @@ -103,10 +105,16 @@ class MountTestBase(unittest.TestCase): t0 = time.time() self.llfuse_thread.join(timeout=60) if self.llfuse_thread.is_alive(): - logger.warning("MountTestBase.tearDown():" - " llfuse thread still alive 60s after umount" - " -- exiting with SIGKILL") - os.kill(os.getpid(), signal.SIGKILL) + # pytest uses exit status 2 when test collection failed. + # A UnitTest failing in setup/teardown counts as a + # collection failure, so pytest will exit with status 2 + # no matter what status you specify here. run-tests.sh + # looks for this status, so specify 2 just to keep + # everything as consistent as possible. + # TODO: If we refactor these tests so they're not built + # on unittest, consider using a dedicated, non-pytest + # exit code like TEMPFAIL. + pytest.exit("llfuse thread outlived test - aborting test suite to avoid deadlock", 2) waited = time.time() - t0 if waited > 0.1: logger.warning("MountTestBase.tearDown(): waited %f s for llfuse thread to end", waited) diff --git a/services/fuse/tests/performance/__init__.py b/services/fuse/tests/performance/__init__.py deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/services/fuse/tests/performance/performance_profiler.py b/services/fuse/tests/performance/performance_profiler.py deleted file mode 120000 index 01a6805b32..0000000000 --- a/services/fuse/tests/performance/performance_profiler.py +++ /dev/null @@ -1 +0,0 @@ -../../../../sdk/python/tests/performance/performance_profiler.py \ No newline at end of file diff --git a/services/fuse/tests/performance/test_collection_performance.py b/services/fuse/tests/performance/test_collection_performance.py deleted file mode 100644 index 525a779ca4..0000000000 --- a/services/fuse/tests/performance/test_collection_performance.py +++ /dev/null @@ -1,488 +0,0 @@ -# Copyright (C) The Arvados Authors. All rights reserved. -# -# SPDX-License-Identifier: AGPL-3.0 - -import arvados -import arvados_fuse as fuse -import llfuse -import logging -import os -import sys -import unittest - -from .. import run_test_server -from ..mount_test_base import MountTestBase -from ..slow_test import slow_test - -logger = logging.getLogger('arvados.arv-mount') - -from .performance_profiler import profiled - -def fuse_createCollectionWithMultipleBlocks(mounttmp, streams=1, files_per_stream=1, data='x'): - class Test(unittest.TestCase): - def runTest(self): - self.createCollectionWithMultipleBlocks() - - @profiled - def createCollectionWithMultipleBlocks(self): - for i in range(0, streams): - os.mkdir(os.path.join(mounttmp, "./stream" + str(i))) - - # Create files - for j in range(0, files_per_stream): - with open(os.path.join(mounttmp, "./stream" + str(i), "file" + str(j) +".txt"), "w") as f: - f.write(data) - - Test().runTest() - -def fuse_readContentsFromCollectionWithMultipleBlocks(mounttmp, streams=1, files_per_stream=1, data='x'): - class Test(unittest.TestCase): - def runTest(self): - self.readContentsFromCollectionWithMultipleBlocks() - - @profiled - def readContentsFromCollectionWithMultipleBlocks(self): - for i in range(0, streams): - d1 = llfuse.listdir(os.path.join(mounttmp, 'stream'+str(i))) - for j in range(0, files_per_stream): - with open(os.path.join(mounttmp, 'stream'+str(i), 'file'+str(i)+'.txt')) as f: - self.assertEqual(data, f.read()) - - Test().runTest() - -def fuse_moveFileFromCollectionWithMultipleBlocks(mounttmp, stream, filename): - class Test(unittest.TestCase): - def runTest(self): - self.moveFileFromCollectionWithMultipleBlocks() - - @profiled - def moveFileFromCollectionWithMultipleBlocks(self): - d1 = llfuse.listdir(os.path.join(mounttmp, stream)) - self.assertIn(filename, d1) - - os.rename(os.path.join(mounttmp, stream, filename), os.path.join(mounttmp, 'moved_from_'+stream+'_'+filename)) - - d1 = llfuse.listdir(os.path.join(mounttmp)) - self.assertIn('moved_from_'+stream+'_'+filename, d1) - - d1 = llfuse.listdir(os.path.join(mounttmp, stream)) - self.assertNotIn(filename, d1) - - Test().runTest() - -def fuse_deleteFileFromCollectionWithMultipleBlocks(mounttmp, stream, filename): - class Test(unittest.TestCase): - def runTest(self): - self.deleteFileFromCollectionWithMultipleBlocks() - - @profiled - def deleteFileFromCollectionWithMultipleBlocks(self): - os.remove(os.path.join(mounttmp, stream, filename)) - - Test().runTest() - -# Create a collection with 2 streams, 3 files_per_stream, 2 blocks_per_file, 2**26 bytes_per_block -class CreateCollectionWithMultipleBlocksAndMoveAndDeleteFile(MountTestBase): - def setUp(self): - super(CreateCollectionWithMultipleBlocksAndMoveAndDeleteFile, self).setUp() - - @slow_test - def test_CreateCollectionWithManyBlocksAndMoveAndDeleteFile(self): - collection = arvados.collection.Collection(api_client=self.api) - collection.save_new() - - m = self.make_mount(fuse.CollectionDirectory) - with llfuse.lock: - m.new_collection(collection.api_response(), collection) - self.assertTrue(m.writable()) - - streams = 2 - files_per_stream = 3 - blocks_per_file = 2 - bytes_per_block = 2**26 - - data = 'x' * blocks_per_file * bytes_per_block - - self.pool.apply(fuse_createCollectionWithMultipleBlocks, (self.mounttmp, streams, files_per_stream, data,)) - - collection2 = self.api.collections().get(uuid=collection.manifest_locator()).execute() - - for i in range(0, streams): - self.assertIn('./stream' + str(i), collection2["manifest_text"]) - - for i in range(0, files_per_stream): - self.assertIn('file' + str(i) + '.txt', collection2["manifest_text"]) - - # Read file contents - self.pool.apply(fuse_readContentsFromCollectionWithMultipleBlocks, (self.mounttmp, streams, files_per_stream, data,)) - - # Move file0.txt out of the streams into . - for i in range(0, streams): - self.pool.apply(fuse_moveFileFromCollectionWithMultipleBlocks, (self.mounttmp, 'stream'+str(i), 'file0.txt',)) - - collection2 = self.api.collections().get(uuid=collection.manifest_locator()).execute() - - manifest_streams = collection2['manifest_text'].split('\n') - self.assertEqual(4, len(manifest_streams)) - - for i in range(0, streams): - self.assertIn('file0.txt', manifest_streams[0]) - - for i in range(0, streams): - self.assertNotIn('file0.txt', manifest_streams[i+1]) - - for i in range(0, streams): - for j in range(1, files_per_stream): - self.assertIn('file' + str(j) + '.txt', manifest_streams[i+1]) - - # Delete 'file1.txt' from all the streams - for i in range(0, streams): - self.pool.apply(fuse_deleteFileFromCollectionWithMultipleBlocks, (self.mounttmp, 'stream'+str(i), 'file1.txt')) - - collection2 = self.api.collections().get(uuid=collection.manifest_locator()).execute() - - manifest_streams = collection2['manifest_text'].split('\n') - self.assertEqual(4, len(manifest_streams)) - - for i in range(0, streams): - self.assertIn('file0.txt', manifest_streams[0]) - - self.assertNotIn('file1.txt', collection2['manifest_text']) - - for i in range(0, streams): - for j in range(2, files_per_stream): - self.assertIn('file' + str(j) + '.txt', manifest_streams[i+1]) - - -def fuse_createCollectionWithManyFiles(mounttmp, streams=1, files_per_stream=1, data='x'): - class Test(unittest.TestCase): - def runTest(self): - self.createCollectionWithManyFiles() - - @profiled - def createCollectionWithManyFiles(self): - for i in range(0, streams): - os.mkdir(os.path.join(mounttmp, "./stream" + str(i))) - - # Create files - for j in range(0, files_per_stream): - with open(os.path.join(mounttmp, "./stream" + str(i), "file" + str(j) +".txt"), "w") as f: - f.write(data) - - Test().runTest() - -def fuse_readContentsFromCollectionWithManyFiles(mounttmp, streams=1, files_per_stream=1, data='x'): - class Test(unittest.TestCase): - def runTest(self): - self.readContentsFromCollectionWithManyFiles() - - @profiled - def readContentsFromCollectionWithManyFiles(self): - for i in range(0, streams): - d1 = llfuse.listdir(os.path.join(mounttmp, 'stream'+str(i))) - for j in range(0, files_per_stream): - with open(os.path.join(mounttmp, 'stream'+str(i), 'file'+str(i)+'.txt')) as f: - self.assertEqual(data, f.read()) - - Test().runTest() - -def fuse_moveFileFromCollectionWithManyFiles(mounttmp, stream, filename): - class Test(unittest.TestCase): - def runTest(self): - self.moveFileFromCollectionWithManyFiles() - - @profiled - def moveFileFromCollectionWithManyFiles(self): - d1 = llfuse.listdir(os.path.join(mounttmp, stream)) - self.assertIn(filename, d1) - - os.rename(os.path.join(mounttmp, stream, filename), os.path.join(mounttmp, 'moved_from_'+stream+'_'+filename)) - - d1 = llfuse.listdir(os.path.join(mounttmp)) - self.assertIn('moved_from_'+stream+'_'+filename, d1) - - d1 = llfuse.listdir(os.path.join(mounttmp, stream)) - self.assertNotIn(filename, d1) - - Test().runTest() - -def fuse_deleteFileFromCollectionWithManyFiles(mounttmp, stream, filename): - class Test(unittest.TestCase): - def runTest(self): - self.deleteFileFromCollectionWithManyFiles() - - @profiled - def deleteFileFromCollectionWithManyFiles(self): - os.remove(os.path.join(mounttmp, stream, filename)) - - Test().runTest() - -# Create a collection with two streams, each with 200 files -class CreateCollectionWithManyFilesAndMoveAndDeleteFile(MountTestBase): - def setUp(self): - super(CreateCollectionWithManyFilesAndMoveAndDeleteFile, self).setUp() - - @slow_test - def test_CreateCollectionWithManyFilesAndMoveAndDeleteFile(self): - collection = arvados.collection.Collection(api_client=self.api) - collection.save_new() - - m = self.make_mount(fuse.CollectionDirectory) - with llfuse.lock: - m.new_collection(collection.api_response(), collection) - self.assertTrue(m.writable()) - - streams = 2 - files_per_stream = 200 - data = 'x' - - self.pool.apply(fuse_createCollectionWithManyFiles, (self.mounttmp, streams, files_per_stream, data,)) - - collection2 = self.api.collections().get(uuid=collection.manifest_locator()).execute() - - for i in range(0, streams): - self.assertIn('./stream' + str(i), collection2["manifest_text"]) - - for i in range(0, files_per_stream): - self.assertIn('file' + str(i) + '.txt', collection2["manifest_text"]) - - # Read file contents - self.pool.apply(fuse_readContentsFromCollectionWithManyFiles, (self.mounttmp, streams, files_per_stream, data,)) - - # Move file0.txt out of the streams into . - for i in range(0, streams): - self.pool.apply(fuse_moveFileFromCollectionWithManyFiles, (self.mounttmp, 'stream'+str(i), 'file0.txt',)) - - collection2 = self.api.collections().get(uuid=collection.manifest_locator()).execute() - - manifest_streams = collection2['manifest_text'].split('\n') - self.assertEqual(4, len(manifest_streams)) - - for i in range(0, streams): - self.assertIn('file0.txt', manifest_streams[0]) - - for i in range(0, streams): - self.assertNotIn('file0.txt', manifest_streams[i+1]) - - for i in range(0, streams): - for j in range(1, files_per_stream): - self.assertIn('file' + str(j) + '.txt', manifest_streams[i+1]) - - # Delete 'file1.txt' from all the streams - for i in range(0, streams): - self.pool.apply(fuse_deleteFileFromCollectionWithManyFiles, (self.mounttmp, 'stream'+str(i), 'file1.txt')) - - collection2 = self.api.collections().get(uuid=collection.manifest_locator()).execute() - - manifest_streams = collection2['manifest_text'].split('\n') - self.assertEqual(4, len(manifest_streams)) - - for i in range(0, streams): - self.assertIn('file0.txt', manifest_streams[0]) - - self.assertNotIn('file1.txt', collection2['manifest_text']) - - for i in range(0, streams): - for j in range(2, files_per_stream): - self.assertIn('file' + str(j) + '.txt', manifest_streams[i+1]) - - -def magicDirTest_MoveFileFromCollection(mounttmp, collection1, collection2, stream, filename): - class Test(unittest.TestCase): - def runTest(self): - self.magicDirTest_moveFileFromCollection() - - @profiled - def magicDirTest_moveFileFromCollection(self): - os.rename(os.path.join(mounttmp, collection1, filename), os.path.join(mounttmp, collection2, filename)) - - Test().runTest() - -def magicDirTest_RemoveFileFromCollection(mounttmp, collection1, stream, filename): - class Test(unittest.TestCase): - def runTest(self): - self.magicDirTest_removeFileFromCollection() - - @profiled - def magicDirTest_removeFileFromCollection(self): - os.remove(os.path.join(mounttmp, collection1, filename)) - - Test().runTest() - -class UsingMagicDir_CreateCollectionWithManyFilesAndMoveAndDeleteFile(MountTestBase): - def setUp(self): - super(UsingMagicDir_CreateCollectionWithManyFilesAndMoveAndDeleteFile, self).setUp() - - @profiled - def magicDirTest_createCollectionWithManyFiles(self, streams=0, files_per_stream=0, data='x'): - # Create collection - collection = arvados.collection.Collection(api_client=self.api) - for j in range(0, files_per_stream): - with collection.open("file"+str(j)+".txt", "w") as f: - f.write(data) - collection.save_new() - return collection - - @profiled - def magicDirTest_readCollectionContents(self, collection, streams=1, files_per_stream=1, data='x'): - mount_ls = os.listdir(os.path.join(self.mounttmp, collection)) - - files = {} - for j in range(0, files_per_stream): - files[os.path.join(self.mounttmp, collection, 'file'+str(j)+'.txt')] = data - - for k, v in files.items(): - with open(os.path.join(self.mounttmp, collection, k)) as f: - self.assertEqual(v, f.read()) - - @slow_test - def test_UsingMagicDirCreateCollectionWithManyFilesAndMoveAndDeleteFile(self): - streams = 2 - files_per_stream = 200 - data = 'x' - - collection1 = self.magicDirTest_createCollectionWithManyFiles() - # Create collection with multiple files - collection2 = self.magicDirTest_createCollectionWithManyFiles(streams, files_per_stream, data) - - # Mount FuseMagicDir - self.make_mount(fuse.MagicDirectory) - - self.magicDirTest_readCollectionContents(collection2.manifest_locator(), streams, files_per_stream, data) - - # Move file0.txt out of the collection2 into collection1 - self.pool.apply(magicDirTest_MoveFileFromCollection, (self.mounttmp, collection2.manifest_locator(), - collection1.manifest_locator(), 'stream0', 'file0.txt',)) - updated_collection = self.api.collections().get(uuid=collection2.manifest_locator()).execute() - self.assertFalse('file0.txt' in updated_collection['manifest_text']) - self.assertTrue('file1.txt' in updated_collection['manifest_text']) - - # Delete file1.txt from collection2 - self.pool.apply(magicDirTest_RemoveFileFromCollection, (self.mounttmp, collection2.manifest_locator(), 'stream0', 'file1.txt',)) - updated_collection = self.api.collections().get(uuid=collection2.manifest_locator()).execute() - self.assertFalse('file1.txt' in updated_collection['manifest_text']) - self.assertTrue('file2.txt' in updated_collection['manifest_text']) - - -def magicDirTest_MoveAllFilesFromCollection(mounttmp, from_collection, to_collection, stream, files_per_stream): - class Test(unittest.TestCase): - def runTest(self): - self.magicDirTest_moveAllFilesFromCollection() - - @profiled - def magicDirTest_moveAllFilesFromCollection(self): - for j in range(0, files_per_stream): - os.rename(os.path.join(mounttmp, from_collection, 'file'+str(j)+'.txt'), os.path.join(mounttmp, to_collection, 'file'+str(j)+'.txt')) - - Test().runTest() - -class UsingMagicDir_CreateCollectionWithManyFilesAndMoveAllFilesIntoAnother(MountTestBase): - def setUp(self): - super(UsingMagicDir_CreateCollectionWithManyFilesAndMoveAllFilesIntoAnother, self).setUp() - - @profiled - def magicDirTestMoveAllFiles_createCollectionWithManyFiles(self, streams=0, files_per_stream=0, - blocks_per_file=0, bytes_per_block=0, data='x'): - # Create collection - collection = arvados.collection.Collection(api_client=self.api) - for j in range(0, files_per_stream): - with collection.open("file"+str(j)+".txt", "w") as f: - f.write(data) - collection.save_new() - return collection - - @slow_test - def test_UsingMagicDirCreateCollectionWithManyFilesAndMoveAllFilesIntoAnother(self): - streams = 2 - files_per_stream = 200 - data = 'x' - - collection1 = self.magicDirTestMoveAllFiles_createCollectionWithManyFiles() - # Create collection with multiple files - collection2 = self.magicDirTestMoveAllFiles_createCollectionWithManyFiles(streams, files_per_stream, data) - - # Mount FuseMagicDir - self.make_mount(fuse.MagicDirectory) - - # Move all files from collection2 into collection1 - self.pool.apply(magicDirTest_MoveAllFilesFromCollection, (self.mounttmp, collection2.manifest_locator(), - collection1.manifest_locator(), 'stream0', files_per_stream,)) - - updated_collection = self.api.collections().get(uuid=collection2.manifest_locator()).execute() - file_names = ["file%i.txt" % i for i in range(0, files_per_stream)] - for name in file_names: - self.assertFalse(name in updated_collection['manifest_text']) - - updated_collection = self.api.collections().get(uuid=collection1.manifest_locator()).execute() - for name in file_names: - self.assertTrue(name in updated_collection['manifest_text']) - - -# Move one file at a time from one collection into another -class UsingMagicDir_CreateCollectionWithManyFilesAndMoveEachFileIntoAnother(MountTestBase): - def setUp(self): - super(UsingMagicDir_CreateCollectionWithManyFilesAndMoveEachFileIntoAnother, self).setUp() - - @profiled - def magicDirTestMoveFiles_createCollectionWithManyFiles(self, streams=0, files_per_stream=0, data='x'): - # Create collection - collection = arvados.collection.Collection(api_client=self.api) - for j in range(0, files_per_stream): - with collection.open("file"+str(j)+".txt", "w") as f: - f.write(data) - collection.save_new() - return collection - - def magicDirTestMoveFiles_oneEachIntoAnother(self, from_collection, to_collection, files_per_stream): - for j in range(0, files_per_stream): - self.pool.apply(magicDirTest_MoveFileFromCollection, (self.mounttmp, from_collection.manifest_locator(), - to_collection.manifest_locator(), 'stream0', 'file'+str(j)+'.txt',)) - - @slow_test - def test_UsingMagicDirCreateCollectionWithManyFilesAndMoveEachFileIntoAnother(self): - streams = 2 - files_per_stream = 200 - data = 'x' - - collection1 = self.magicDirTestMoveFiles_createCollectionWithManyFiles() - # Create collection with multiple files - collection2 = self.magicDirTestMoveFiles_createCollectionWithManyFiles(streams, files_per_stream, data) - - # Mount FuseMagicDir - self.make_mount(fuse.MagicDirectory) - - # Move all files from collection2 into collection1 - self.magicDirTestMoveFiles_oneEachIntoAnother(collection2, collection1, files_per_stream) - - updated_collection = self.api.collections().get(uuid=collection2.manifest_locator()).execute() - file_names = ["file%i.txt" % i for i in range(0, files_per_stream)] - for name in file_names: - self.assertFalse(name in updated_collection['manifest_text']) - - updated_collection = self.api.collections().get(uuid=collection1.manifest_locator()).execute() - for name in file_names: - self.assertTrue(name in updated_collection['manifest_text']) - -class FuseListLargeProjectContents(MountTestBase): - @profiled - def getProjectWithManyCollections(self): - project_contents = llfuse.listdir(self.mounttmp) - self.assertEqual(201, len(project_contents)) - self.assertIn('Collection_1', project_contents) - - @profiled - def listContentsInProjectWithManyCollections(self): - project_contents = llfuse.listdir(self.mounttmp) - self.assertEqual(201, len(project_contents)) - self.assertIn('Collection_1', project_contents) - - for collection_name in project_contents: - collection_contents = llfuse.listdir(os.path.join(self.mounttmp, collection_name)) - self.assertIn('baz', collection_contents) - - @slow_test - def test_listLargeProjectContents(self): - self.make_mount(fuse.ProjectDirectory, - project_object=run_test_server.fixture('groups')['project_with_201_collections']) - self.getProjectWithManyCollections() - self.listContentsInProjectWithManyCollections() diff --git a/services/fuse/tests/slow_test.py b/services/fuse/tests/slow_test.py deleted file mode 120000 index c7e1f7fada..0000000000 --- a/services/fuse/tests/slow_test.py +++ /dev/null @@ -1 +0,0 @@ -../../../sdk/python/tests/slow_test.py \ No newline at end of file diff --git a/tools/crunchstat-summary/arvados_version.py b/tools/crunchstat-summary/arvados_version.py index 794b6afe42..ca20de4c58 100644 --- a/tools/crunchstat-summary/arvados_version.py +++ b/tools/crunchstat-summary/arvados_version.py @@ -72,14 +72,6 @@ if REPO_PATH is None: if (SETUP_DIR / mod_name).is_dir() ) -def short_tests_only(arglist=sys.argv): - try: - arglist.remove('--short-tests-only') - except ValueError: - return False - else: - return True - def git_log_output(path, *args): return subprocess.check_output( ['git', '-C', str(REPO_PATH), diff --git a/tools/crunchstat-summary/pytest.ini b/tools/crunchstat-summary/pytest.ini new file mode 120000 index 0000000000..05a82dbfef --- /dev/null +++ b/tools/crunchstat-summary/pytest.ini @@ -0,0 +1 @@ +../../sdk/python/pytest.ini \ No newline at end of file diff --git a/tools/crunchstat-summary/setup.py b/tools/crunchstat-summary/setup.py index 95c534c90b..d5adc92367 100755 --- a/tools/crunchstat-summary/setup.py +++ b/tools/crunchstat-summary/setup.py @@ -11,7 +11,6 @@ from setuptools import setup, find_packages import arvados_version version = arvados_version.get_version() -short_tests_only = arvados_version.short_tests_only() README = os.path.join(arvados_version.SETUP_DIR, 'README.rst') setup(name='crunchstat_summary', diff --git a/tools/user-activity/arvados_version.py b/tools/user-activity/arvados_version.py index 794b6afe42..ca20de4c58 100644 --- a/tools/user-activity/arvados_version.py +++ b/tools/user-activity/arvados_version.py @@ -72,14 +72,6 @@ if REPO_PATH is None: if (SETUP_DIR / mod_name).is_dir() ) -def short_tests_only(arglist=sys.argv): - try: - arglist.remove('--short-tests-only') - except ValueError: - return False - else: - return True - def git_log_output(path, *args): return subprocess.check_output( ['git', '-C', str(REPO_PATH), diff --git a/tools/user-activity/pytest.ini b/tools/user-activity/pytest.ini new file mode 120000 index 0000000000..05a82dbfef --- /dev/null +++ b/tools/user-activity/pytest.ini @@ -0,0 +1 @@ +../../sdk/python/pytest.ini \ No newline at end of file