sdk/python/tests/fed-migrate/*.cwlex
doc/install/*.xlsx
sdk/cwl/tests/wf/hello.txt
-sdk/cwl/tests/wf/indir1/hello2.txt
\ No newline at end of file
+sdk/cwl/tests/wf/indir1/hello2.txt
+sdk/cwl/tests/chipseq/data/Genomes/*
\ No newline at end of file
See https://docs.nvidia.com/deploy/cuda-compatibility/ for
details.
- cudaComputeCapabilityMin:
- type: string
- doc: Minimum CUDA hardware capability required to run the software, in X.Y format.
- deviceCountMin:
- type: int?
+ cudaComputeCapability:
+ type:
+ - 'string'
+ - 'string[]'
+ doc: |
+ CUDA hardware capability required to run the software, in X.Y
+ format.
+
+ * If this is a single value, it defines only the minimum
+ compute capability. GPUs with higher capability are also
+ accepted.
+
+ * If it is an array value, then only select GPUs with compute
+ capabilities that explicitly appear in the array.
+ cudaDeviceCountMin:
+ type: ['null', int, cwl:Expression]
default: 1
- doc: Minimum number of GPU devices to request, default 1.
- deviceCountMax:
- type: int?
- doc: Maximum number of GPU devices to request. If not specified, same as `deviceCountMin`.
+ doc: |
+ Minimum number of GPU devices to request. If not specified,
+ same as `cudaDeviceCountMax`. If neither are specified,
+ default 1.
+ cudaDeviceCountMax:
+ type: ['null', int, cwl:Expression]
+ doc: |
+ Maximum number of GPU devices to request. If not specified,
+ same as `cudaDeviceCountMin`.
See https://docs.nvidia.com/deploy/cuda-compatibility/ for
details.
- cudaComputeCapabilityMin:
- type: string
- doc: Minimum CUDA hardware capability required to run the software, in X.Y format.
- deviceCountMin:
- type: int?
+ cudaComputeCapability:
+ type:
+ - 'string'
+ - 'string[]'
+ doc: |
+ CUDA hardware capability required to run the software, in X.Y
+ format.
+
+ * If this is a single value, it defines only the minimum
+ compute capability. GPUs with higher capability are also
+ accepted.
+
+ * If it is an array value, then only select GPUs with compute
+ capabilities that explicitly appear in the array.
+ cudaDeviceCountMin:
+ type: ['null', int, cwl:Expression]
default: 1
- doc: Minimum number of GPU devices to request, default 1.
- deviceCountMax:
- type: int?
- doc: Maximum number of GPU devices to request. If not specified, same as `deviceCountMin`.
+ doc: |
+ Minimum number of GPU devices to request. If not specified,
+ same as `cudaDeviceCountMax`. If neither are specified,
+ default 1.
+ cudaDeviceCountMax:
+ type: ['null', int, cwl:Expression]
+ doc: |
+ Maximum number of GPU devices to request. If not specified,
+ same as `cudaDeviceCountMin`.
See https://docs.nvidia.com/deploy/cuda-compatibility/ for
details.
- cudaComputeCapabilityMin:
- type: string
- doc: Minimum CUDA hardware capability required to run the software, in X.Y format.
- deviceCountMin:
- type: int?
+ cudaComputeCapability:
+ type:
+ - 'string'
+ - 'string[]'
+ doc: |
+ CUDA hardware capability required to run the software, in X.Y
+ format.
+
+ * If this is a single value, it defines only the minimum
+ compute capability. GPUs with higher capability are also
+ accepted.
+
+ * If it is an array value, then only select GPUs with compute
+ capabilities that explicitly appear in the array.
+ cudaDeviceCountMin:
+ type: ['null', int, cwl:Expression]
default: 1
- doc: Minimum number of GPU devices to request, default 1.
- deviceCountMax:
- type: int?
- doc: Maximum number of GPU devices to request. If not specified, same as `deviceCountMin`.
+ doc: |
+ Minimum number of GPU devices to request. If not specified,
+ same as `cudaDeviceCountMax`. If neither are specified,
+ default 1.
+ cudaDeviceCountMax:
+ type: ['null', int, cwl:Expression]
+ doc: |
+ Maximum number of GPU devices to request. If not specified,
+ same as `cudaDeviceCountMin`.
cuda_req, _ = self.get_requirement("http://commonwl.org/cwltool#CUDARequirement")
if cuda_req:
runtime_constraints["cuda"] = {
- "device_count": cuda_req.get("deviceCountMin", 1),
+ "device_count": resources.get("cudaDeviceCount", 1),
"driver_version": cuda_req["cudaVersionMin"],
- "hardware_capability": cuda_req["cudaComputeCapabilityMin"]
+ "hardware_capability": aslist(cuda_req["cudaComputeCapability"])[0]
}
if self.timelimit is not None and self.timelimit > 0:
# file to determine what version of cwltool and schema-salad to
# build.
install_requires=[
- 'cwltool==3.1.20220217222804',
+ 'cwltool==3.1.20220224085855',
'schema-salad==8.2.20211116214159',
'arvados-python-client{}'.format(pysdk_dep),
'setuptools',
--- /dev/null
+{
+ "referenceGenomeSequence": {
+ "class": "File",
+ "location": "data/Genomes/Homo_sapiens/GRCh38.p2/WholeGenome/genome.fa",
+ "metadata": {
+ "reference_genome": {
+ "organism": "Homo sapiens",
+ "version": "hg38"
+ },
+ "annotation": {
+ "source": "gencode",
+ "version": "v24"
+ }
+ }
+ },
+ "referenceGenomeSequenceDrosophila": {
+ "class": "File",
+ "location": "data/Genomes/Drosophila_melanogaster/dmel_r6.16/WholeGenome/genome.fa",
+ "metadata": {
+ "reference_genome": {
+ "organism": "Drosophila melanogaster",
+ "version": "rmel_r6.16"
+ }
+ }
+ },
+ "blacklistBed": {
+ "class": "File",
+ "location": "data/Genomes/Blacklist/lists2/hg38-blacklist.v2.bed",
+ "metadata": {
+ "reference_genome": {
+ "organism": "Homo sapiens",
+ "version": "hg38"
+ },
+ "annotation": {
+ "source": "gencode",
+ "version": "v24"
+ }
+ }
+ },
+ "BowtieHumanReference": {
+ "class": "Directory",
+ "location": "data/Genomes/Homo_sapiens/GRCh38.p2/Bowtie2Index/",
+ "metadata": {
+ "reference_genome": {
+ "organism": "Homo sapiens",
+ "version": "hg38"
+ },
+ "annotation": {
+ "source": "gencode",
+ "version": "v24"
+ }
+ }
+ },
+ "BowtieDrosophilaReference": {
+ "class": "Directory",
+ "location": "data/Genomes/Drosophila_melanogaster/dmel_r6.16/Bowtie2Index/",
+ "metadata": {
+ "reference_genome": {
+ "organism": "Drosophila melanogaster",
+ "version": "rmel_r6.16"
+ }
+ }
+ },
+ "sampleName": "LED054_0p03nMR1.0",
+ "inputFastq1": {
+ "class": "File",
+ "metadata": {
+ "user": "kmavrommatis",
+ "sample_id": [
+ 2
+ ]
+ },
+ "location": "DATEST/ChIP-Seq/Raw/fastq/Input_R1.fastq.gz",
+ "secondaryFiles": []
+ },
+ "inputFastq2": {
+ "class": "File",
+ "metadata": {
+ "user": "kmavrommatis",
+ "sample_id": [
+ 2
+ ]
+ },
+ "location": "DATEST/ChIP-Seq/Raw/fastq/Input_R3.fastq.gz",
+ "secondaryFiles": []
+ },
+ "inputFastqUMI": {
+ "class": "File",
+ "metadata": {
+ "user": "kmavrommatis",
+ "sample_id": [
+ 2
+ ]
+ },
+ "location": "DATEST/ChIP-Seq/Raw/fastq/Input_R2.fastq.gz",
+ "secondaryFiles": []
+ }
+}
+
--- /dev/null
+{
+ "$graph": [
+ {
+ "class": "Workflow",
+ "id": "#main",
+ "doc": "Pipeline that is applied on single ChIP-seq samples.\n\nStarts with QC on the reads and trimming (for adapters and based on quality)\n\nAligns to human genome and adds UMI\n\nAligns to Drosophila genome and counts the number of reads.\n\nAfter the alignment to human genome the files are filtered for duplicates, multimappers and alignments in black listed regions",
+ "label": "ChIP-Seq (single sample)",
+ "inputs": [
+ {
+ "id": "#inputFastq1",
+ "type": "File",
+ "https://www.sevenbridges.com/fileTypes": "fastq",
+ "https://www.sevenbridges.com/x": 0,
+ "https://www.sevenbridges.com/y": 1726.25
+ },
+ {
+ "id": "#blacklistBed",
+ "type": "File",
+ "https://www.sevenbridges.com/x": 746.4744873046875,
+ "https://www.sevenbridges.com/y": 1903.265625
+ },
+ {
+ "id": "#referenceGenomeSequence",
+ "type": "File",
+ "secondaryFiles": [
+ ".fai",
+ "^.dict"
+ ],
+ "https://www.sevenbridges.com/fileTypes": "fasta, fa",
+ "https://www.sevenbridges.com/x": 0,
+ "https://www.sevenbridges.com/y": 1405.203125
+ },
+ {
+ "id": "#sampleName",
+ "type": "string",
+ "https://www.sevenbridges.com/x": 0,
+ "https://www.sevenbridges.com/y": 1191.171875
+ },
+ {
+ "id": "#inputFastq2",
+ "type": [
+ "null",
+ "File"
+ ],
+ "https://www.sevenbridges.com/fileTypes": "fastq",
+ "https://www.sevenbridges.com/x": 0,
+ "https://www.sevenbridges.com/y": 1619.234375
+ },
+ {
+ "id": "#inputFastqUMI",
+ "type": "File",
+ "https://www.sevenbridges.com/x": 0,
+ "https://www.sevenbridges.com/y": 1512.21875
+ },
+ {
+ "id": "#BowtieHumanReference",
+ "type": "Directory",
+ "https://www.sevenbridges.com/x": 363.875,
+ "https://www.sevenbridges.com/y": 1519.21875
+ },
+ {
+ "id": "#BowtieDrosophilaReference",
+ "type": "Directory",
+ "https://www.sevenbridges.com/x": 363.875,
+ "https://www.sevenbridges.com/y": 1626.234375
+ },
+ {
+ "id": "#referenceGenomeSequenceDrosophila",
+ "type": "File",
+ "secondaryFiles": [
+ ".fai"
+ ],
+ "https://www.sevenbridges.com/x": 0,
+ "https://www.sevenbridges.com/y": 1298.1875
+ }
+ ],
+ "outputs": [
+ ],
+ "steps": [
+ {
+ "id": "#step1",
+ "in": {
+ "inp": "#inputFastq1"
+ },
+ "out": [],
+ "run": "../cat.cwl"
+ }
+ ],
+ "requirements": [
+ ]
+ },
+ ],
+ "cwlVersion": "v1.0"
+}
runner.api.collections().get().execute.return_value = {
"portable_data_hash": "99999999999999999999999999999993+99"}
- tool = cmap({
- "inputs": [],
- "outputs": [],
- "baseCommand": "nvidia-smi",
- "arguments": [],
- "id": "",
- "cwlVersion": "v1.2",
- "class": "CommandLineTool",
- "requirements": [
- {
+ test_cwl_req = [{
"class": "http://commonwl.org/cwltool#CUDARequirement",
"cudaVersionMin": "11.0",
- "cudaComputeCapabilityMin": "9.0",
- }
- ]
- })
+ "cudaComputeCapability": "9.0",
+ }, {
+ "class": "http://commonwl.org/cwltool#CUDARequirement",
+ "cudaVersionMin": "11.0",
+ "cudaComputeCapability": "9.0",
+ "cudaDeviceCountMin": 2
+ }, {
+ "class": "http://commonwl.org/cwltool#CUDARequirement",
+ "cudaVersionMin": "11.0",
+ "cudaComputeCapability": ["4.0", "5.0"],
+ "cudaDeviceCountMin": 2
+ }]
+
+ test_arv_req = [{
+ 'device_count': 1,
+ 'driver_version': "11.0",
+ 'hardware_capability': "9.0"
+ }, {
+ 'device_count': 2,
+ 'driver_version': "11.0",
+ 'hardware_capability': "9.0"
+ }, {
+ 'device_count': 2,
+ 'driver_version': "11.0",
+ 'hardware_capability': "4.0"
+ }]
+
+ for test_case in range(0, len(test_cwl_req)):
- loadingContext, runtimeContext = self.helper(runner, True)
+ tool = cmap({
+ "inputs": [],
+ "outputs": [],
+ "baseCommand": "nvidia-smi",
+ "arguments": [],
+ "id": "",
+ "cwlVersion": "v1.2",
+ "class": "CommandLineTool",
+ "requirements": [test_cwl_req[test_case]]
+ })
- arvtool = cwltool.load_tool.load_tool(tool, loadingContext)
- arvtool.formatgraph = None
+ loadingContext, runtimeContext = self.helper(runner, True)
- for j in arvtool.job({}, mock.MagicMock(), runtimeContext):
- j.run(runtimeContext)
- runner.api.container_requests().create.assert_called_with(
- body=JsonDiffMatcher({
- 'environment': {
- 'HOME': '/var/spool/cwl',
- 'TMPDIR': '/tmp'
- },
- 'name': 'test_run_True',
- 'runtime_constraints': {
- 'vcpus': 1,
- 'ram': 268435456,
- 'cuda': {
- 'device_count': 1,
- 'driver_version': "11.0",
- 'hardware_capability': "9.0"
- }
- },
- 'use_existing': True,
- 'priority': 500,
- 'mounts': {
- '/tmp': {'kind': 'tmp',
- "capacity": 1073741824
- },
- '/var/spool/cwl': {'kind': 'tmp',
- "capacity": 1073741824 }
- },
- 'state': 'Committed',
- 'output_name': 'Output for step test_run_True',
- 'owner_uuid': 'zzzzz-8i9sb-zzzzzzzzzzzzzzz',
- 'output_path': '/var/spool/cwl',
- 'output_ttl': 0,
- 'container_image': '99999999999999999999999999999993+99',
- 'command': ['nvidia-smi'],
- 'cwd': '/var/spool/cwl',
- 'scheduling_parameters': {},
- 'properties': {},
- 'secret_mounts': {},
- 'output_storage_classes': ["default"]
- }))
+ arvtool = cwltool.load_tool.load_tool(tool, loadingContext)
+ arvtool.formatgraph = None
+
+ for j in arvtool.job({}, mock.MagicMock(), runtimeContext):
+ j.run(runtimeContext)
+ runner.api.container_requests().create.assert_called_with(
+ body=JsonDiffMatcher({
+ 'environment': {
+ 'HOME': '/var/spool/cwl',
+ 'TMPDIR': '/tmp'
+ },
+ 'name': 'test_run_True' + ("" if test_case == 0 else "_"+str(test_case+1)),
+ 'runtime_constraints': {
+ 'vcpus': 1,
+ 'ram': 268435456,
+ 'cuda': test_arv_req[test_case]
+ },
+ 'use_existing': True,
+ 'priority': 500,
+ 'mounts': {
+ '/tmp': {'kind': 'tmp',
+ "capacity": 1073741824
+ },
+ '/var/spool/cwl': {'kind': 'tmp',
+ "capacity": 1073741824 }
+ },
+ 'state': 'Committed',
+ 'output_name': 'Output for step test_run_True' + ("" if test_case == 0 else "_"+str(test_case+1)),
+ 'owner_uuid': 'zzzzz-8i9sb-zzzzzzzzzzzzzzz',
+ 'output_path': '/var/spool/cwl',
+ 'output_ttl': 0,
+ 'container_image': '99999999999999999999999999999993+99',
+ 'command': ['nvidia-smi'],
+ 'cwd': '/var/spool/cwl',
+ 'scheduling_parameters': {},
+ 'properties': {},
+ 'secret_mounts': {},
+ 'output_storage_classes': ["default"]
+ }))
# The test passes no builder.resources
projects / arvados.git / commitdiff