--upload
If the build and test steps are successful, upload the packages
to a remote apt repository (default: false)
+--rc
+ Optional Parameter to build Release Candidate
--build-version <version>
Version to build (default:
\$ARVADOS_BUILDING_VERSION-\$ARVADOS_BUILDING_ITERATION or
fi
PARSEDOPTS=$(getopt --name "$0" --longoptions \
- help,upload,target:,build-version: \
+ help,upload,rc,target:,build-version: \
-- "" "$@")
if [ $? -ne 0 ]; then
exit 1
TARGET=debian8
UPLOAD=0
+RC=0
declare -a build_args=()
--upload)
UPLOAD=1
;;
+ --rc)
+ RC=1
+ ;;
--build-version)
build_args+=("$1" "$2")
shift
timer_reset
if [ ${#failures[@]} -eq 0 ]; then
- echo "/usr/local/arvados-dev/jenkins/run_upload_packages.py -H jenkinsapt@apt.arvados.org -o Port=2222 --workspace $WORKSPACE $TARGET"
- /usr/local/arvados-dev/jenkins/run_upload_packages.py -H jenkinsapt@apt.arvados.org -o Port=2222 --workspace $WORKSPACE $TARGET
+ if [[ "$RC" != 0 ]]; then
+ echo "/usr/local/arvados-dev/jenkins/run_upload_packages_testing.py -H jenkinsapt@apt.arvados.org -o Port=2222 --workspace $WORKSPACE $TARGET"
+ /usr/local/arvados-dev/jenkins/run_upload_packages_testing.py -H jenkinsapt@apt.arvados.org -o Port=2222 --workspace $WORKSPACE $TARGET
+ else
+ echo "/usr/local/arvados-dev/jenkins/run_upload_packages.py -H jenkinsapt@apt.arvados.org -o Port=2222 --workspace $WORKSPACE $TARGET"
+ /usr/local/arvados-dev/jenkins/run_upload_packages.py -H jenkinsapt@apt.arvados.org -o Port=2222 --workspace $WORKSPACE $TARGET
+ fi
else
echo "Skipping package upload, there were errors building and/or testing the packages"
fi
title "End of upload packages (`timer`)"
fi
-exit_cleanly
+exit_cleanly
\ No newline at end of file
gem 'themes_for_rails', git: 'https://github.com/curoverse/themes_for_rails'
-gem 'arvados', '>= 0.1.20150615153458'
+gem 'arvados', '>= 1.3.1.20190301212059'
gem 'httpclient'
gem 'sshkey'
activemodel (>= 3.0.0)
activesupport (>= 3.0.0)
rack (>= 1.1.0)
- addressable (2.5.2)
+ addressable (2.6.0)
public_suffix (>= 2.0.2, < 4.0)
andand (1.3.3)
arel (6.0.4)
- arvados (0.1.20180302192246)
+ arvados (1.3.1.20190301212059)
activesupport (>= 3)
andand (~> 1.3, >= 1.3.3)
- google-api-client (>= 0.7, < 0.8.9)
+ cure-google-api-client (>= 0.7, < 0.8.9)
i18n (~> 0)
json (>= 1.7.7, < 3)
jwt (>= 0.1.5, < 2)
- arvados-cli (1.1.4.20180723133344)
+ arvados-cli (1.3.1.20190211211047)
activesupport (>= 3.2.13, < 5)
andand (~> 1.3, >= 1.3.3)
- arvados (~> 0.1, >= 0.1.20150128223554)
+ arvados (~> 1.3.0, >= 1.3.0)
curb (~> 0.8)
- google-api-client (~> 0.6, >= 0.6.3, < 0.8.9)
+ cure-google-api-client (~> 0.6, >= 0.6.3, < 0.8.9)
json (>= 1.7.7, < 3)
oj (~> 3.0)
- trollop (~> 2.0)
+ optimist (~> 3.0)
autoparse (0.3.3)
addressable (>= 2.3.1)
extlib (>= 0.9.15)
coffee-script-source (1.12.2)
concurrent-ruby (1.1.4)
crass (1.0.4)
- curb (0.9.6)
+ curb (0.9.8)
+ cure-google-api-client (0.8.7.1)
+ activesupport (>= 3.2, < 5.0)
+ addressable (~> 2.3)
+ autoparse (~> 0.3)
+ extlib (~> 0.9)
+ faraday (~> 0.9)
+ googleauth (~> 0.3)
+ launchy (~> 2.4)
+ multi_json (~> 1.10)
+ retriable (~> 1.4)
+ signet (~> 0.6)
database_cleaner (1.7.0)
erubis (2.7.0)
eventmachine (1.2.6)
websocket-driver (>= 0.5.1)
globalid (0.4.1)
activesupport (>= 4.2.0)
- google-api-client (0.8.7)
- activesupport (>= 3.2, < 5.0)
- addressable (~> 2.3)
- autoparse (~> 0.3)
- extlib (~> 0.9)
- faraday (~> 0.9)
- googleauth (~> 0.3)
- launchy (~> 2.4)
- multi_json (~> 1.10)
- retriable (~> 1.4)
- signet (~> 0.6)
- googleauth (0.6.2)
+ googleauth (0.8.0)
faraday (~> 0.12)
jwt (>= 1.4, < 3.0)
- logging (~> 2.0)
- memoist (~> 0.12)
+ memoist (~> 0.16)
multi_json (~> 1.11)
- os (~> 0.9)
+ os (>= 0.9, < 2.0)
signet (~> 0.7)
hashie (3.5.7)
highline (1.7.10)
rails-dom-testing (>= 1, < 3)
railties (>= 4.2.0)
thor (>= 0.14, < 2.0)
- json (2.1.0)
+ json (2.2.0)
jwt (1.5.6)
launchy (2.4.3)
addressable (~> 2.3)
libv8 (3.16.14.19)
- little-plugger (1.1.4)
- logging (2.2.2)
- little-plugger (~> 1.1)
- multi_json (~> 1.10)
lograge (0.10.0)
actionpack (>= 4)
activesupport (>= 4)
multi_json (~> 1.3)
multi_xml (~> 0.5)
rack (>= 1.2, < 3)
- oj (3.6.4)
+ oj (3.7.9)
omniauth (1.4.3)
hashie (>= 1.2, < 4)
rack (>= 1.6.2, < 3)
omniauth-oauth2 (1.5.0)
oauth2 (~> 1.1)
omniauth (~> 1.2)
- os (0.9.6)
+ optimist (3.0.0)
+ os (1.0.0)
passenger (5.3.0)
rack
rake (>= 0.8.1)
power_assert (1.1.1)
protected_attributes (1.1.4)
activemodel (>= 4.0.1, < 5.0)
- public_suffix (3.0.2)
+ public_suffix (3.0.3)
rack (1.6.11)
rack-test (0.6.3)
rack (>= 1.0)
sass (~> 3.2.2)
sprockets (~> 2.8, < 3.0)
sprockets-rails (~> 2.0)
- signet (0.8.1)
+ signet (0.11.0)
addressable (~> 2.3)
faraday (~> 0.9)
jwt (>= 1.5, < 3.0)
thor (0.20.3)
thread_safe (0.3.6)
tilt (1.4.1)
- trollop (2.1.2)
+ trollop (2.9.9)
tzinfo (1.2.5)
thread_safe (~> 0.1)
uglifier (2.7.2)
activerecord-deprecated_finders
acts_as_api
andand
- arvados (>= 0.1.20150615153458)
+ arvados (>= 1.3.1.20190301212059)
arvados-cli
coffee-rails (~> 4.0)
database_cleaner
# SPDX-License-Identifier: AGPL-3.0
require 'whitelist_update'
+require 'arvados/collection'
class ContainerRequest < ArvadosModel
include ArvadosModelUpdates
coll = Collection.new(
owner_uuid: self.owner_uuid,
name: coll_name,
+ manifest_text: "",
properties: {
'type' => out_type,
'container_request' => uuid,
})
end
+
+ if out_type == "log"
+ src = Arv::Collection.new(manifest)
+ dst = Arv::Collection.new(coll.manifest_text)
+ dst.cp_r("./", ".", src)
+ dst.cp_r("./", "log for container #{container.uuid}", src)
+ manifest = dst.manifest_text
+ end
+
coll.assign_attributes(
- portable_data_hash: pdh,
+ portable_data_hash: Digest::MD5.hexdigest(manifest) + '+' + manifest.bytesize.to_s,
manifest_text: manifest,
trash_at: trash_at,
delete_at: trash_at)
return false
else
self.container_count += 1
+ if self.container_uuid_was
+ old_container = Container.find_by_uuid(self.container_uuid_was)
+ old_logs = Collection.where(portable_data_hash: old_container.log).first
+ if old_logs
+ log_coll = self.log_uuid.nil? ? nil : Collection.where(uuid: self.log_uuid).first
+ if self.log_uuid.nil?
+ log_coll = Collection.new(
+ owner_uuid: self.owner_uuid,
+ name: coll_name = "Container log for request #{uuid}",
+ manifest_text: "")
+ end
+
+ # copy logs from old container into CR's log collection
+ src = Arv::Collection.new(old_logs.manifest_text)
+ dst = Arv::Collection.new(log_coll.manifest_text)
+ dst.cp_r("./", "log for container #{old_container.uuid}", src)
+ manifest = dst.manifest_text
+
+ log_coll.assign_attributes(
+ portable_data_hash: Digest::MD5.hexdigest(manifest) + '+' + manifest.bytesize.to_s,
+ manifest_text: manifest)
+ log_coll.save_with_unique_name!
+ self.log_uuid = log_coll.uuid
+ end
+ end
end
end
end
require 'test_helper'
require 'helpers/container_test_helper'
require 'helpers/docker_migration_helper'
+require 'arvados/collection'
class ContainerRequestTest < ActiveSupport::TestCase
include DockerMigrationHelper
cr.reload
assert_equal "Final", cr.state
assert_equal users(:active).uuid, cr.modified_by_user_uuid
- ['output', 'log'].each do |out_type|
- pdh = Container.find_by_uuid(cr.container_uuid).send(out_type)
- assert_equal(1, Collection.where(portable_data_hash: pdh,
- owner_uuid: project.uuid).count,
- "Container #{out_type} should be copied to #{project.uuid}")
- end
+
assert_not_nil cr.output_uuid
assert_not_nil cr.log_uuid
output = Collection.find_by_uuid cr.output_uuid
assert_equal output_pdh, output.portable_data_hash
+ assert_equal output.owner_uuid, project.uuid, "Container output should be copied to #{project.uuid}"
+
log = Collection.find_by_uuid cr.log_uuid
- assert_equal log_pdh, log.portable_data_hash
+ assert_equal log.manifest_text, ". 37b51d194a7513e45b56f6524f2d51f2+3 0:3:bar
+./log\\040for\\040container\\040#{cr.container_uuid} 37b51d194a7513e45b56f6524f2d51f2+3 0:3:bar\n"
+
+ assert_equal log.owner_uuid, project.uuid, "Container log should be copied to #{project.uuid}"
end
test "Container makes container request, then is cancelled" do
cr.reload
assert_equal "Final", cr.state
assert_equal prev_container_uuid, cr.container_uuid
+ end
+
+
+ test "Retry saves logs from previous attempts" do
+ set_user_from_auth :active
+ cr = create_minimal_req!(priority: 1, state: "Committed", container_count_max: 3)
+
+ c = act_as_system_user do
+ c = Container.find_by_uuid(cr.container_uuid)
+ c.update_attributes!(state: Container::Locked)
+ c.update_attributes!(state: Container::Running)
+ c
+ end
+
+ container_uuids = []
+
+ [0, 1, 2].each do
+ cr.reload
+ assert_equal "Committed", cr.state
+ container_uuids << cr.container_uuid
+
+ c = act_as_system_user do
+ logc = Collection.new(manifest_text: ". 37b51d194a7513e45b56f6524f2d51f2+3 0:3:bar\n")
+ logc.save!
+ c = Container.find_by_uuid(cr.container_uuid)
+ c.update_attributes!(state: Container::Cancelled, log: logc.portable_data_hash)
+ c
+ end
+ end
+
+ container_uuids.sort!
+
+ cr.reload
+ assert_equal "Final", cr.state
+ assert_equal 3, cr.container_count
+ assert_equal ". 37b51d194a7513e45b56f6524f2d51f2+3 0:3:bar
+./log\\040for\\040container\\040#{container_uuids[0]} 37b51d194a7513e45b56f6524f2d51f2+3 0:3:bar
+./log\\040for\\040container\\040#{container_uuids[1]} 37b51d194a7513e45b56f6524f2d51f2+3 0:3:bar
+./log\\040for\\040container\\040#{container_uuids[2]} 37b51d194a7513e45b56f6524f2d51f2+3 0:3:bar
+" , Collection.find_by_uuid(cr.log_uuid).manifest_text
end
cr2.reload
assert_equal cr1log_uuid, cr1.log_uuid
assert_equal cr2log_uuid, cr2.log_uuid
- assert_equal [logpdh_time2], Collection.where(uuid: [cr1log_uuid, cr2log_uuid]).to_a.collect(&:portable_data_hash).uniq
+ assert_equal 1, Collection.where(uuid: [cr1log_uuid, cr2log_uuid]).to_a.collect(&:portable_data_hash).uniq.length
+ assert_equal ". acbd18db4cc2f85cedef654fccc4a4d8+3 cdd549ae79fe6640fa3d5c6261d8303c+195 0:3:foo.txt 3:195:zzzzz-8i9sb-0vsrcqi7whchuil.log.txt
+./log\\040for\\040container\\040#{cr1.container_uuid} acbd18db4cc2f85cedef654fccc4a4d8+3 cdd549ae79fe6640fa3d5c6261d8303c+195 0:3:foo.txt 3:195:zzzzz-8i9sb-0vsrcqi7whchuil.log.txt
+", Collection.find_by_uuid(cr1log_uuid).manifest_text
end
["auth_uuid", "runtime_token"].each do |tok|
cw.write("data 8")
cw.start_new_stream('edgecases')
- for f in ":/.../-/*/\x01\\/ ".split("/"):
+ for f in ":/.../-/*/ ".split("/"):
cw.start_new_file(f)
cw.write('x')
- for f in ":/.../-/*/\x01\\/ ".split("/"):
+ for f in ":/.../-/*/ ".split("/"):
cw.start_new_stream('edgecases/dirs/' + f)
cw.start_new_file('x/x')
cw.write('x')
self.assertDirContents('dir2', ['thing5.txt', 'thing6.txt', 'dir3'])
self.assertDirContents('dir2/dir3', ['thing7.txt', 'thing8.txt'])
self.assertDirContents('edgecases',
- "dirs/:/.../-/*/\x01\\/ ".split("/"))
+ "dirs/:/.../-/*/ ".split("/"))
self.assertDirContents('edgecases/dirs',
- ":/.../-/*/\x01\\/ ".split("/"))
+ ":/.../-/*/ ".split("/"))
files = {'thing1.txt': 'data 1',
'thing2.txt': 'data 2',
projects / arvados.git / commitdiff