// For example, filterable.js writes filters in
// infiniteContentParamsFilterable ("search for text foo")
// without worrying about clobbering the filters set up by the
- // tab pane ("only show jobs and pipelines in this tab").
+ // tab pane ("only show container requests and pipeline instances
+ // in this tab").
$.each($container.data(), function(datakey, datavalue) {
// Note: We attach these data to DOM elements using
// <element data-foo-bar="baz">. We store/retrieve them
# Column names should always be qualified by a table name and a direction is optional, defaulting to asc
# (e.g. "collections.name" or "collections.name desc").
# If a column name is specified, that table will be sorted by that column.
- # If there are objects from different models that will be shown (such as in Jobs and Pipelines tab),
+ # If there are objects from different models that will be shown (such as in Pipelines and processes tab),
# then a sort column name can optionally be specified for each model, passed as an comma-separated list (e.g. "jobs.script, pipeline_instances.name")
# Currently only one sort column name and direction can be specified for each model.
def load_filters_and_paging_params
class ContainerRequestsController < ApplicationController
+ skip_around_filter :require_thread_api_token, if: proc { |ctrl|
+ Rails.configuration.anonymous_user_token and
+ 'show' == ctrl.action_name
+ }
+
def show_pane_list
%w(Status Log Advanced)
end
class ContainersController < ApplicationController
+ skip_around_filter :require_thread_api_token, if: proc { |ctrl|
+ Rails.configuration.anonymous_user_token and
+ 'show' == ctrl.action_name
+ }
+
def show_pane_list
%w(Status Log Advanced)
end
}
pane_list <<
{
- :name => 'Jobs_and_pipelines',
- :filters => [%w(uuid is_a) + [%w(arvados#job arvados#pipelineInstance)]]
+ :name => 'Pipelines_and_processes',
+ :filters => [%w(uuid is_a) + [%w(arvados#containerRequest arvados#pipelineInstance)]]
}
pane_list <<
{
@name_link_for = {}
kind_filters.each do |attr,op,val|
(val.is_a?(Array) ? val : [val]).each do |type|
+ filters = @filters - kind_filters + [['uuid', 'is_a', type]]
+ if type == 'arvados#containerRequest'
+ filters = filters + [['container_requests.requesting_container_uuid', '=', nil]]
+ end
objects = @object.contents(order: @order,
limit: @limit,
- filters: (@filters - kind_filters + [['uuid', 'is_a', type]]),
+ filters: filters,
)
objects.each do |object|
@name_link_for[object.andand.uuid] = objects.links_for(object, 'name').first
[ 'description' ]
end
+ def self.goes_in_projects?
+ true
+ end
+
def work_unit(label=nil)
ContainerWorkUnit.new(self, label)
end
+++ /dev/null
-<%= render_pane 'tab_contents', to_string: true, locals: {
- limit: 50,
- filters: [['uuid', 'is_a', ["arvados#job", "arvados#pipelineInstance"]]],
- sortable_columns: { 'name' => 'jobs.script, pipeline_instances.name', 'description' => 'jobs.description, pipeline_instances.description' }
- }.merge(local_assigns) %>
--- /dev/null
+<%= render_pane 'tab_contents', to_string: true, locals: {
+ limit: 50,
+ filters: [['uuid', 'is_a', ["arvados#containerRequest", "arvados#pipelineInstance"]]],
+ sortable_columns: { 'name' => 'container_requests.name, pipeline_instances.name', 'description' => 'container_requests.description, pipeline_instances.description' }
+ }.merge(local_assigns) %>
[
["active", 5, ["aproject", "asubproject"], "anonymously_accessible_project"],
- ["user1_with_load", 2, ["project_with_10_collections"], "project_with_2_pipelines_and_60_jobs"],
+ ["user1_with_load", 2, ["project_with_10_collections"], "project_with_2_pipelines_and_60_crs"],
["admin", 5, ["anonymously_accessible_project", "subproject_in_anonymous_accessible_project"], "aproject"],
].each do |user, page_size, tree_segment, unexpected|
test "build my projects tree for #{user} user and verify #{unexpected} is omitted" do
assert_selector 'a', text: 'Description'
assert_selector 'a', text: 'Data collections'
- assert_selector 'a', text: 'Jobs and pipelines'
+ assert_selector 'a', text: 'Pipelines and processes'
assert_selector 'a', text: 'Pipeline templates'
assert_selector 'a', text: 'Subprojects'
assert_selector 'a', text: 'Advanced'
end
[
- 'running_job',
- 'completed_job',
+ 'running anonymously accessible cr',
'pipelineInstance'
- ].each do |type|
- test "anonymous user accesses jobs and pipelines tab in shared project and clicks on #{type}" do
+ ].each do |proc|
+ test "anonymous user accesses pipelines and processes tab in shared project and clicks on '#{proc}'" do
visit PUBLIC_PROJECT
click_link 'Data collections'
assert_text 'GNU General Public License'
- click_link 'Jobs and pipelines'
+ click_link 'Pipelines and processes'
assert_text 'Pipeline in publicly accessible project'
- # click on the specified job
- if type.include? 'job'
- verify_job_row type
- else
+ if proc.include? 'pipeline'
verify_pipeline_instance_row
+ else
+ verify_container_request_row proc
end
end
end
- def verify_job_row look_for
+ def verify_container_request_row look_for
within first('tr', text: look_for) do
click_link 'Show'
end
assert_text 'Public Projects Unrestricted public data'
- assert_text 'script_version'
+ assert_text 'command'
assert_text 'zzzzz-tpzed-xurymjxw79nv3jz' # modified by user
assert_no_selector 'a', text: 'zzzzz-tpzed-xurymjxw79nv3jz'
- assert_no_selector 'a', text: 'Move job'
assert_no_selector 'button', text: 'Cancel'
- assert_no_selector 'button', text: 'Re-run job'
end
def verify_pipeline_instance_row
wait_for_ajax
end
- click_link 'Jobs and pipelines'
+ click_link 'Pipelines and processes'
find('tr[data-kind="arvados#pipelineInstance"]', text: '(none)').
find('a', text: 'Show').
click
[
['project_with_10_pipelines', 10, 0],
- ['project_with_2_pipelines_and_60_jobs', 2, 60],
+ ['project_with_2_pipelines_and_60_crs', 2, 60],
['project_with_25_pipelines', 25, 0],
- ].each do |project_name, num_pipelines, num_jobs|
- test "scroll pipeline instances tab for #{project_name} with #{num_pipelines} pipelines and #{num_jobs} jobs" do
- item_list_parameter = "Jobs_and_pipelines"
+ ].each do |project_name, num_pipelines, num_crs|
+ test "scroll pipeline instances tab for #{project_name} with #{num_pipelines} pipelines and #{num_crs} container requests" do
+ item_list_parameter = "Pipelines_and_processes"
scroll_setup project_name,
- num_pipelines + num_jobs,
+ num_pipelines + num_crs,
item_list_parameter
# check the general scrolling and the pipelines
scroll_items_check num_pipelines,
"pipeline_",
item_list_parameter,
'tr[data-kind="arvados#pipelineInstance"]'
- # Check job count separately
- jobs_found = page.all('tr[data-kind="arvados#job"]')
- found_job_count = jobs_found.count
- assert_equal num_jobs, found_job_count, 'Did not find expected number of jobs'
+ # Check container request count separately
+ crs_found = page.all('tr[data-kind="arvados#containerRequest"]')
+ found_cr_count = crs_found.count
+ assert_equal num_crs, found_cr_count, 'Did not find expected number of container requests'
end
end
assert_no_selector 'li.disabled', text: 'Copy selected'
end
- # Go to Jobs and pipelines tab and assert none selected
- click_link 'Jobs and pipelines'
+ # Go to Pipelines and processes tab and assert none selected
+ click_link 'Pipelines and processes'
wait_for_ajax
# Since this is the first visit to this tab, all selection options should be disabled
Note: Because adding access tokens to manifests can be computationally expensive, the @manifest_text@ field is not included in listed collections. If you need it, request a "list of collections":{{site.baseurl}}/api/methods/collections.html with the filter @["owner_uuid", "=", GROUP_UUID]@, and @"manifest_text"@ listed in the select parameter.
+Note: Use filters with the attribute format @<item type>.<field name>@ to filter items of a specific type. For example: @["pipeline_instances.state", "=", "Complete"]@ to filter @pipeline_instances@ where @state@ is @Complete@. All other types of items owned by this group will be unimpacted by this filter and will still be included.
+
h2. create
Create a new Group.
<notextile>
<pre><code>~$ <span class="userinput">keepstore -h</span>
-2015/05/08 13:41:16 keepstore starting, pid 2565
+2016/07/01 14:06:21 keepstore starting, pid 32339
Usage of ./keepstore:
- -azure-storage-account-key-file="": File containing the account key used for subsequent --azure-storage-container-volume arguments.
- -azure-storage-account-name="": Azure storage account name used for subsequent --azure-storage-container-volume arguments.
- -azure-storage-container-volume=[]: Use the given container as a storage volume. Can be given multiple times.
- -azure-storage-replication=3: Replication level to report to clients when data is stored in an Azure container.
- -blob-signature-ttl=1209600: Lifetime of blob permission signatures. Modifying the ttl will invalidate all existing signatures. See services/api/config/application.default.yml.
- -blob-signing-key-file="": File containing the secret key for generating and verifying blob permission signatures.
- -data-manager-token-file="": File with the API token used by the Data Manager. All DELETE requests or GET /index requests must carry this token.
- -enforce-permissions=false: Enforce permission signatures on requests.
- -listen=":25107": Listening address, in the form "host:port". e.g., 10.0.1.24:8000. Omit the host part to listen on all interfaces.
- -max-buffers=128: Maximum RAM to use for data buffers, given in multiples of block size (64 MiB). When this limit is reached, HTTP requests requiring buffers (like GET and PUT) will wait for buffer space to be released.
+ -azure-max-get-bytes int
+ Maximum bytes to request in a single GET request. If smaller than 67108864, use multiple concurrent range requests to retrieve a block. (default 67108864)
+ -azure-storage-account-key-file string
+ File containing the account key used for subsequent --azure-storage-container-volume arguments.
+ -azure-storage-account-name string
+ Azure storage account name used for subsequent --azure-storage-container-volume arguments.
+ -azure-storage-container-volume value
+ Use the given container as a storage volume. Can be given multiple times. (default [])
+ -azure-storage-replication int
+ Replication level to report to clients when data is stored in an Azure container. (default 3)
+ -blob-signature-ttl int
+ Lifetime of blob permission signatures in seconds. Modifying the ttl will invalidate all existing signatures. See services/api/config/application.default.yml. (default 1209600)
+ -blob-signing-key-file string
+ File containing the secret key for generating and verifying blob permission signatures.
+ -data-manager-token-file string
+ File with the API token used by the Data Manager. All DELETE requests or GET /index requests must carry this token.
+ -enforce-permissions
+ Enforce permission signatures on requests.
+ -listen string
+ Listening address, in the form "host:port". e.g., 10.0.1.24:8000. Omit the host part to listen on all interfaces. (default ":25107")
+ -max-buffers int
+ Maximum RAM to use for data buffers, given in multiples of block size (64 MiB). When this limit is reached, HTTP requests requiring buffers (like GET and PUT) will wait for buffer space to be released. (default 128)
-max-requests int
- Maximum concurrent requests. When this limit is reached, new requests will receive 503 responses. Note: this limit does not include idle connections from clients using HTTP keepalive, so it does not strictly limit the number of concurrent connections. (default 2 * max-buffers)
- -never-delete=false: If set, nothing will be deleted. HTTP 405 will be returned for valid DELETE requests.
- -permission-key-file="": Synonym for -blob-signing-key-file.
- -permission-ttl=0: Synonym for -blob-signature-ttl.
- -pid="": Path to write pid file during startup. This file is kept open and locked with LOCK_EX until keepstore exits, so `fuser -k pidfile` is one way to shut down. Exit immediately if there is an error opening, locking, or writing the pid file.
- -readonly=false: Do not write, delete, or touch anything on the following volumes.
- -serialize=false: Serialize read and write operations on the following volumes.
- -volume=[]: Local storage directory. Can be given more than once to add multiple directories. If none are supplied, the default is to use all directories named "keep" that exist in the top level directory of a mount point at startup time. Can be a comma-separated list, but this is deprecated: use multiple -volume arguments instead.
- -volumes=[]: Deprecated synonym for -volume.
+ Maximum concurrent requests. When this limit is reached, new requests will receive 503 responses. Note: this limit does not include idle connections from clients using HTTP keepalive, so it does not strictly limit the number of concurrent connections. (default 2 * max-buffers)
+ -never-delete
+ If true, nothing will be deleted. Warning: the relevant features in keepstore and data manager have not been extensively tested. You should leave this option alone unless you can afford to lose data. (default true)
+ -permission-key-file string
+ Synonym for -blob-signing-key-file.
+ -permission-ttl int
+ Synonym for -blob-signature-ttl.
+ -pid fuser -k pidfile
+ Path to write pid file during startup. This file is kept open and locked with LOCK_EX until keepstore exits, so fuser -k pidfile is one way to shut down. Exit immediately if there is an error opening, locking, or writing the pid file.
+ -readonly
+ Do not write, delete, or touch anything on the following volumes.
+ -s3-access-key-file string
+ File containing the access key used for subsequent -s3-bucket-volume arguments.
+ -s3-bucket-volume value
+ Use the given bucket as a storage volume. Can be given multiple times. (default [])
+ -s3-endpoint string
+ Endpoint URL used for subsequent -s3-bucket-volume arguments. If blank, use the AWS endpoint corresponding to the -s3-region argument. For Google Storage, use "https://storage.googleapis.com".
+ -s3-region string
+ AWS region used for subsequent -s3-bucket-volume arguments. Allowed values are ["ap-southeast-1" "eu-west-1" "us-gov-west-1" "sa-east-1" "cn-north-1" "ap-northeast-1" "ap-southeast-2" "eu-central-1" "us-east-1" "us-west-1" "us-west-2"].
+ -s3-replication int
+ Replication level reported to clients for subsequent -s3-bucket-volume arguments. (default 2)
+ -s3-secret-key-file string
+ File containing the secret key used for subsequent -s3-bucket-volume arguments.
+ -s3-unsafe-delete
+ EXPERIMENTAL. Enable deletion (garbage collection), even though there are known race conditions that can cause data loss.
+ -serialize
+ Serialize read and write operations on the following volumes.
+ -trash-check-interval duration
+ Time duration at which the emptyTrash goroutine will check and delete expired trashed blocks. Default is one day. (default 24h0m0s)
+ -trash-lifetime duration
+ Time duration after a block is trashed during which it can be recovered using an /untrash request
+ -volume value
+ Local storage directory. Can be given more than once to add multiple directories. If none are supplied, the default is to use all directories named "keep" that exist in the top level directory of a mount point at startup time. Can be a comma-separated list, but this is deprecated: use multiple -volume arguments instead. (default [])
+ -volumes value
+ Deprecated synonym for -volume. (default [])
</code></pre>
</notextile>
ARG COMMIT=latest
RUN echo $COMMIT && apt-get update -q
-RUN apt-get install -qy git python-pip python-virtualenv python-arvados-python-client python-dev libcurl4-gnutls-dev nodejs python-arvados-cwl-runner
+RUN apt-get install -qy git python-pip python-virtualenv python-arvados-python-client python-dev libgnutls28-dev libcurl4-gnutls-dev nodejs python-arvados-cwl-runner
# Install dependencies and set up system.
RUN /usr/sbin/adduser --disabled-password \
},
},
{
- in: map[string]interface{}{"foo": map[string]interface{}{"bar":1.234}},
+ in: map[string]interface{}{"foo": map[string]interface{}{"bar": 1.234}},
ok: func(out url.Values) bool {
return out.Get("foo") == `{"bar":1.234}`
},
// us about a stored block.
type KeepServiceIndexEntry struct {
SizedDigest
+ // Time of last write, in nanoseconds since Unix epoch
Mtime int64
}
if err != nil {
return nil, fmt.Errorf("Malformed index line %q: mtime: %v", line, err)
}
+ if mtime < 1e12 {
+ // An old version of keepstore is giving us
+ // timestamps in seconds instead of
+ // nanoseconds. (This threshold correctly
+ // handles all times between 1970-01-02 and
+ // 33658-09-27.)
+ mtime = mtime * 1e9
+ }
entries = append(entries, KeepServiceIndexEntry{
SizedDigest: SizedDigest(fields[0]),
Mtime: mtime,
"zzzz-ot0gb-111111111111111",
tmpdir,
"",
- Job{Script_parameters: Tasks{[]TaskDef{TaskDef{
+ Job{Script_parameters: Tasks{[]TaskDef{{
Command: []string{"echo", "foo"}}}}},
Task{Sequence: 0})
c.Check(err, IsNil)
tmpdir,
"",
Job{Script_parameters: Tasks{[]TaskDef{
- TaskDef{Command: []string{"echo", "bar"}},
- TaskDef{Command: []string{"echo", "foo"}}}}},
+ {Command: []string{"echo", "bar"}},
+ {Command: []string{"echo", "foo"}}}}},
Task{Parameters: TaskDef{
Command: []string{"echo", "foo"},
Stdout: "output.txt"},
"zzzz-ot0gb-111111111111111",
tmpdir,
"",
- Job{Script_parameters: Tasks{[]TaskDef{TaskDef{
+ Job{Script_parameters: Tasks{[]TaskDef{{
Command: []string{"cat"},
Stdout: "output.txt",
Stdin: tmpfile.Name()}}}},
"zzzz-ot0gb-111111111111111",
tmpdir,
"",
- Job{Script_parameters: Tasks{[]TaskDef{TaskDef{
+ Job{Script_parameters: Tasks{[]TaskDef{{
Command: []string{"/bin/sh", "-c", "echo $BAR"},
Stdout: "output.txt",
Env: map[string]string{"BAR": "foo"}}}}},
"zzzz-ot0gb-111111111111111",
tmpdir,
"foo\n",
- Job{Script_parameters: Tasks{[]TaskDef{TaskDef{
+ Job{Script_parameters: Tasks{[]TaskDef{{
Command: []string{"/bin/sh", "-c", "echo $BAR"},
Stdout: "output.txt",
Env: map[string]string{"BAR": "$(task.keep)"}}}}},
"zzzz-ot0gb-111111111111111",
tmpdir,
"",
- Job{Script_parameters: Tasks{[]TaskDef{TaskDef{
+ Job{Script_parameters: Tasks{[]TaskDef{{
Command: []string{"/bin/sh", "-c", "echo $PATH"},
Stdout: "output.txt",
Env: map[string]string{"PATH": "foo"}}}}},
func (s *TestSuite) TestScheduleSubtask(c *C) {
api := SubtaskTestClient{c, []Task{
- Task{Job_uuid: "zzzz-8i9sb-111111111111111",
+ {Job_uuid: "zzzz-8i9sb-111111111111111",
Created_by_job_task_uuid: "zzzz-ot0gb-111111111111111",
Sequence: 1,
Parameters: TaskDef{
Command: []string{"echo", "bar"}}},
- Task{Job_uuid: "zzzz-8i9sb-111111111111111",
+ {Job_uuid: "zzzz-8i9sb-111111111111111",
Created_by_job_task_uuid: "zzzz-ot0gb-111111111111111",
Sequence: 1,
Parameters: TaskDef{
tmpdir,
"",
Job{Script_parameters: Tasks{[]TaskDef{
- TaskDef{Command: []string{"echo", "bar"}},
- TaskDef{Command: []string{"echo", "foo"}}}}},
+ {Command: []string{"echo", "bar"}},
+ {Command: []string{"echo", "foo"}}}}},
Task{Sequence: 0})
c.Check(err, IsNil)
"zzzz-ot0gb-111111111111111",
tmpdir,
"",
- Job{Script_parameters: Tasks{[]TaskDef{TaskDef{
+ Job{Script_parameters: Tasks{[]TaskDef{{
Command: []string{"/bin/sh", "-c", "exit 1"}}}}},
Task{Sequence: 0})
c.Check(err, FitsTypeOf, PermFail{})
"zzzz-ot0gb-111111111111111",
tmpdir,
"",
- Job{Script_parameters: Tasks{[]TaskDef{TaskDef{
+ Job{Script_parameters: Tasks{[]TaskDef{{
Command: []string{"/bin/sh", "-c", "exit 1"},
SuccessCodes: []int{0, 1}}}}},
Task{Sequence: 0})
"zzzz-ot0gb-111111111111111",
tmpdir,
"",
- Job{Script_parameters: Tasks{[]TaskDef{TaskDef{
+ Job{Script_parameters: Tasks{[]TaskDef{{
Command: []string{"/bin/sh", "-c", "exit 0"},
PermanentFailCodes: []int{0, 1}}}}},
Task{Sequence: 0})
"zzzz-ot0gb-111111111111111",
tmpdir,
"",
- Job{Script_parameters: Tasks{[]TaskDef{TaskDef{
+ Job{Script_parameters: Tasks{[]TaskDef{{
Command: []string{"/bin/sh", "-c", "exit 1"},
TemporaryFailCodes: []int{1}}}}},
Task{Sequence: 0})
"zzzz-ot0gb-111111111111111",
tmpdir,
"",
- Job{Script_parameters: Tasks{[]TaskDef{TaskDef{
+ Job{Script_parameters: Tasks{[]TaskDef{{
Command: []string{"ls", "output.txt"},
Vwd: map[string]string{
"output.txt": tmpfile.Name()}}}}},
"zzzz-ot0gb-111111111111111",
tmpdir,
keepmount,
- Job{Script_parameters: Tasks{[]TaskDef{TaskDef{
+ Job{Script_parameters: Tasks{[]TaskDef{{
Command: []string{"cat"},
Stdout: "output.txt",
Stdin: "$(task.keep)/file1.txt"}}}},
"zzzz-ot0gb-111111111111111",
tmpdir,
keepmount,
- Job{Script_parameters: Tasks{[]TaskDef{TaskDef{
+ Job{Script_parameters: Tasks{[]TaskDef{{
Command: []string{"cat", "$(task.keep)/file1.txt"},
Stdout: "output.txt"}}}},
Task{Sequence: 0})
"zzzz-ot0gb-111111111111111",
tmpdir,
"",
- Job{Script_parameters: Tasks{[]TaskDef{TaskDef{
+ Job{Script_parameters: Tasks{[]TaskDef{{
Command: []string{"sleep", "4"}}}}},
Task{Sequence: 0})
c.Check(err, FitsTypeOf, PermFail{})
"zzzz-ot0gb-111111111111111",
tmpdir,
"",
- Job{Script_parameters: Tasks{[]TaskDef{TaskDef{
+ Job{Script_parameters: Tasks{[]TaskDef{{
Command: []string{"echo", "foo"},
Stdout: "s ub:dir/:e vi\nl"}}}},
Task{Sequence: 0})
// In case we exited the above loop early: before returning,
// drain the toGet channel so its sender doesn't sit around
// blocking forever.
- for _ = range r.toGet {
+ for range r.toGet {
}
}
request_orders = @orders.clone
@orders = []
- [Group,
- Job, PipelineInstance, PipelineTemplate,
+ request_filters = @filters
+
+ klasses = [Group,
+ Job, PipelineInstance, PipelineTemplate, ContainerRequest,
Collection,
- Human, Specimen, Trait].each do |klass|
+ Human, Specimen, Trait]
+
+ table_names = klasses.map(&:table_name)
+ request_filters.each do |col, op, val|
+ if col.index('.') && !table_names.include?(col.split('.', 2)[0])
+ raise ArgumentError.new("Invalid attribute '#{col}' in filter")
+ end
+ end
+
+ klasses.each do |klass|
# If the currently requested orders specifically match the
# table_name for the current klass, apply that order.
# Otherwise, order by recency.
where_conds[:group_class] = "project"
end
+ @filters = request_filters.map do |col, op, val|
+ if !col.index('.')
+ [col, op, val]
+ elsif (col = col.split('.', 2))[0] == klass.table_name
+ [col[1], op, val]
+ else
+ nil
+ end
+ end.compact
+
@objects = klass.readable_by(*@read_users).
order(request_order).where(where_conds)
@limit = limit_all - all_objects.count
output_path: test
command: ["echo", "hello"]
requesting_container_uuid: zzzzz-dz642-requestercntnr1
+
+running_anonymous_accessible:
+ uuid: zzzzz-xvhdp-runninganonaccs
+ owner_uuid: zzzzz-j7d0g-zhxawtyetzwc5f0
+ name: running anonymously accessible cr
+ state: Committed
+ priority: 1
+ created_at: 2016-01-11 11:11:11.111111111 Z
+ updated_at: 2016-01-11 11:11:11.111111111 Z
+ modified_at: 2016-01-11 11:11:11.111111111 Z
+ modified_by_user_uuid: zzzzz-tpzed-xurymjxw79nv3jz
+ container_image: test
+ cwd: test
+ output_path: test
+ command: ["echo", "hello"]
+ container_uuid: zzzzz-dz642-runningcontain2
+
+# Test Helper trims the rest of the file
+
+# Do not add your fixtures below this line as the rest of this file will be trimmed by test_helper
+
+# container requests in project_with_2_pipelines_and_60_crs
+<% for i in 1..60 do %>
+cr_<%=i%>_of_60:
+ uuid: zzzzz-xvhdp-oneof60crs<%= i.to_s.rjust(5, '0') %>
+ created_at: <%= ((i+5)/5).hour.ago.to_s(:db) %>
+ owner_uuid: zzzzz-j7d0g-nnncrspipelines
+ name: cr-<%= i.to_s %>
+ output_path: test
+ command: ["echo", "hello"]
+<% end %>
+
+# Do not add your fixtures below this line as the rest of this file will be trimmed by test_helper
description: project with 10 pipelines
group_class: project
-project_with_2_pipelines_and_60_jobs:
- uuid: zzzzz-j7d0g-nnjobspipelines
+project_with_2_pipelines_and_60_crs:
+ uuid: zzzzz-j7d0g-nnncrspipelines
owner_uuid: zzzzz-tpzed-user1withloadab
created_at: 2014-04-21 15:37:48 -0400
modified_by_client_uuid: zzzzz-ozdt8-brczlopd8u8d0jr
modified_by_user_uuid: zzzzz-tpzed-user1withloadab
modified_at: 2014-04-21 15:37:48 -0400
updated_at: 2014-04-21 15:37:48 -0400
- name: project with 2 pipelines and 60 jobs
+ name: project with 2 pipelines and 60 crs
description: This will result in two pages in the display
group_class: project
components:
component1: zzzzz-8i9sb-jyq01m7in1jlofj
component2: zzzzz-d1hrv-partdonepipelin
-
-# Test Helper trims the rest of the file
-
-# Do not add your fixtures below this line as the rest of this file will be trimmed by test_helper
-
-# jobs in project_with_2_pipelines_and_60_jobs
-<% for i in 1..60 do %>
-job_<%=i%>_of_60:
- uuid: zzzzz-8i9sb-oneof100jobs<%= i.to_s.rjust(3, '0') %>
- created_at: <%= ((i+5)/5).minute.ago.to_s(:db) %>
- owner_uuid: zzzzz-j7d0g-nnjobspipelines
- script_version: 7def43a4d3f20789dda4700f703b5514cc3ed250
- state: Complete
-<% end %>
-
-# Do not add your fixtures below this line as the rest of this file will be trimmed by test_helper
title: foo instance input
<% end %>
-# pipelines in project_with_2_pipelines_and_100_jobs
+# pipelines in project_with_2_pipelines_and_60_crs
<% for i in 1..2 do %>
-pipeline_<%=i%>_of_2_pipelines_and_100_jobs:
+pipeline_<%=i%>_of_2_pipelines_and_60_crs:
name: pipeline_<%= i %>
state: New
uuid: zzzzz-d1hrv-abcgneyn6brx<%= i.to_s.rjust(3, '0') %>
- owner_uuid: zzzzz-j7d0g-nnjobspipelines
+ owner_uuid: zzzzz-j7d0g-nnncrspipelines
created_at: <%= i.minute.ago.to_s(:db) %>
components:
foo:
end
assert_equal true, found_projects.include?(groups(:starred_and_shared_active_user_project).uuid)
end
+
+ [
+ [['owner_uuid', '!=', 'zzzzz-tpzed-xurymjxw79nv3jz'], 200,
+ 'zzzzz-d1hrv-subprojpipeline', 'zzzzz-d1hrv-1xfj6xkicf2muk2'],
+ [["pipeline_instances.state", "not in", ["Complete", "Failed"]], 200,
+ 'zzzzz-d1hrv-1xfj6xkicf2muk2', 'zzzzz-d1hrv-i3e77t9z5y8j9cc'],
+ [['container_requests.requesting_container_uuid', '=', nil], 200,
+ 'zzzzz-xvhdp-cr4queuedcontnr', 'zzzzz-xvhdp-cr4requestercn2'],
+ [['container_requests.no_such_column', '=', nil], 422],
+ [['container_requests.', '=', nil], 422],
+ [['.requesting_container_uuid', '=', nil], 422],
+ [['no_such_table.uuid', '!=', 'zzzzz-tpzed-xurymjxw79nv3jz'], 422],
+ ].each do |filter, expect_code, expect_uuid, not_expect_uuid|
+ test "get contents with '#{filter}' filter" do
+ authorize_with :active
+ get :contents, filters: [filter], format: :json
+ assert_response expect_code
+ if expect_code == 200
+ assert_not_empty json_response['items']
+ item_uuids = json_response['items'].collect {|item| item['uuid']}
+ assert_includes(item_uuids, expect_uuid)
+ assert_not_includes(item_uuids, not_expect_uuid)
+ end
+ end
+ end
end
EM.run {
if token
- ws = Faye::WebSocket::Client.new("ws://localhost:3002/websocket?api_token=#{api_client_authorizations(token).api_token}")
+ ws = Faye::WebSocket::Client.new("ws://localhost:#{WEBSOCKET_PORT}/websocket?api_token=#{api_client_authorizations(token).api_token}")
else
- ws = Faye::WebSocket::Client.new("ws://localhost:3002/websocket")
+ ws = Faye::WebSocket::Client.new("ws://localhost:#{WEBSOCKET_PORT}/websocket")
end
ws.on :open do |event|
require 'bundler'
+require 'socket'
$ARV_API_SERVER_DIR = File.expand_path('../..', __FILE__)
-SERVER_PID_PATH = 'tmp/pids/passenger.3002.pid'
+
+s = TCPServer.new('0.0.0.0', 0)
+WEBSOCKET_PORT = s.addr[1]
+s.close
+SERVER_PID_PATH = "tmp/pids/passenger.#{WEBSOCKET_PORT}.pid"
class WebsocketTestRunner < MiniTest::Unit
def _system(*cmd)
def _run(args=[])
server_pid = Dir.chdir($ARV_API_SERVER_DIR) do |apidir|
# Only passenger seems to be able to run the websockets server successfully.
- _system('passenger', 'start', '-d', '-p3002')
+ _system('passenger', 'start', '-d', "-p#{WEBSOCKET_PORT}")
timeout = Time.now.tv_sec + 10
begin
sleep 0.2
super(args)
ensure
Dir.chdir($ARV_API_SERVER_DIR) do
- _system('passenger', 'stop', '-p3002')
+ _system('passenger', 'stop', "-p#{WEBSOCKET_PORT}")
end
# DatabaseCleaner leaves the database empty. Prefer to leave it full.
dc = DatabaseController.new
}
// drain any subsequent status changes
- for _ = range status {
+ for range status {
}
log.Printf("Finalized container %v", uuid)
// There should be no queued containers now
params := arvadosclient.Dict{
- "filters": [][]string{[]string{"state", "=", "Queued"}},
+ "filters": [][]string{{"state", "=", "Queued"}},
}
var containers arvados.ContainerList
err = arv.List("containers", params, &containers)
// There should be no queued containers now
params := arvadosclient.Dict{
- "filters": [][]string{[]string{"state", "=", "Queued"}},
+ "filters": [][]string{{"state", "=", "Queued"}},
}
var containers arvados.ContainerList
err = arv.List("containers", params, &containers)
signal.Notify(runner.SigChan, syscall.SIGQUIT)
go func(sig <-chan os.Signal) {
- for _ = range sig {
+ for range sig {
if !runner.Cancelled {
runner.CancelLock.Lock()
runner.Cancelled = true
}
line = line[5:]
}
- if len(line) >= 6 && string(line[len(line)-6:len(line)]) == "[...]\n" {
+ if len(line) >= 6 && string(line[len(line)-6:]) == "[...]\n" {
line = line[:len(line)-6]
} else {
done = true
// SdkCollectionInfo holds collection info from api
type SdkCollectionInfo struct {
- UUID string `json:"uuid"`
- OwnerUUID string `json:"owner_uuid"`
- ReplicationDesired int `json:"replication_desired"`
- ModifiedAt time.Time `json:"modified_at"`
- ManifestText string `json:"manifest_text"`
+ UUID string `json:"uuid"`
+ OwnerUUID string `json:"owner_uuid"`
+ ReplicationDesired int `json:"replication_desired"`
+ ModifiedAt time.Time `json:"modified_at"`
+ ManifestText string `json:"manifest_text"`
}
// SdkCollectionList lists collections from api
sdkParams := arvadosclient.Dict{
"select": fieldsWanted,
"order": []string{"modified_at ASC", "uuid ASC"},
- "filters": [][]string{[]string{"modified_at", ">=", "1900-01-01T00:00:00Z"}},
+ "filters": [][]string{{"modified_at", ">=", "1900-01-01T00:00:00Z"}},
"offset": 0}
if params.BatchSize > 0 {
}
func (s *MySuite) TestSummarizeSimple(checker *C) {
- rc := MakeTestReadCollections([]TestCollectionSpec{TestCollectionSpec{
+ rc := MakeTestReadCollections([]TestCollectionSpec{{
ReplicationLevel: 5,
Blocks: []int{1, 2},
}})
expected := ExpectedSummary{
OwnerToCollectionSize: map[string]int{c.OwnerUUID: c.TotalSize},
BlockToDesiredReplication: map[blockdigest.DigestWithSize]int{blockDigest1: 5, blockDigest2: 5},
- BlockToCollectionUuids: map[blockdigest.DigestWithSize][]string{blockDigest1: []string{c.UUID}, blockDigest2: []string{c.UUID}},
+ BlockToCollectionUuids: map[blockdigest.DigestWithSize][]string{blockDigest1: {c.UUID}, blockDigest2: {c.UUID}},
}
CompareSummarizedReadCollections(checker, rc, expected)
func (s *MySuite) TestSummarizeOverlapping(checker *C) {
rc := MakeTestReadCollections([]TestCollectionSpec{
- TestCollectionSpec{
+ {
ReplicationLevel: 5,
Blocks: []int{1, 2},
},
- TestCollectionSpec{
+ {
ReplicationLevel: 8,
Blocks: []int{2, 3},
},
blockDigest3: 8,
},
BlockToCollectionUuids: map[blockdigest.DigestWithSize][]string{
- blockDigest1: []string{c0.UUID},
- blockDigest2: []string{c0.UUID, c1.UUID},
- blockDigest3: []string{c1.UUID},
+ blockDigest1: {c0.UUID},
+ blockDigest2: {c0.UUID, c1.UUID},
+ blockDigest3: {c1.UUID},
},
}
// GetKeepServers from api server
func GetKeepServers(params GetKeepServersParams) (results ReadServers, err error) {
sdkParams := arvadosclient.Dict{
- "filters": [][]string{[]string{"service_type", "!=", "proxy"}},
+ "filters": [][]string{{"service_type", "!=", "proxy"}},
}
if params.Limit > 0 {
sdkParams["limit"] = params.Limit
return
}
- blockInfo.Mtime, err = strconv.ParseInt(tokens[1], 10, 64)
+ var ns int64
+ ns, err = strconv.ParseInt(tokens[1], 10, 64)
if err != nil {
return
}
- blockInfo.Digest =
- blockdigest.DigestWithSize{Digest: locator.Digest,
- Size: uint32(locator.Size)}
+ if ns < 1e12 {
+ // An old version of keepstore is giving us timestamps
+ // in seconds instead of nanoseconds. (This threshold
+ // correctly handles all times between 1970-01-02 and
+ // 33658-09-27.)
+ ns = ns * 1e9
+ }
+ blockInfo.Mtime = ns
+ blockInfo.Digest = blockdigest.DigestWithSize{
+ Digest: locator.Digest,
+ Size: uint32(locator.Size),
+ }
return
}
defer server.Close()
tl := map[string]TrashList{
- server.URL: TrashList{TrashRequest{"000000000000000000000000deadbeef", 99}}}
+ server.URL: {TrashRequest{"000000000000000000000000deadbeef", 99}}}
arv := arvadosclient.ArvadosClient{ApiToken: "abc123"}
kc := keepclient.KeepClient{Arvados: &arv, Client: &http.Client{}}
func sendTrashListError(c *C, server *httptest.Server) {
tl := map[string]TrashList{
- server.URL: TrashList{TrashRequest{"000000000000000000000000deadbeef", 99}}}
+ server.URL: {TrashRequest{"000000000000000000000000deadbeef", 99}}}
arv := arvadosclient.ArvadosClient{ApiToken: "abc123"}
kc := keepclient.KeepClient{Arvados: &arv, Client: &http.Client{}}
locator1 := Locator{Digest: blockdigest.MakeTestBlockDigest(0xBadBeef)}
c.Check(
BuildPullLists(map[Locator]PullServers{
- locator1: PullServers{To: []string{}, From: []string{}}}),
+ locator1: {To: []string{}, From: []string{}}}),
PullListMapEquals,
map[string]PullList{})
c.Check(
BuildPullLists(map[Locator]PullServers{
- locator1: PullServers{To: []string{}, From: []string{"f1", "f2"}}}),
+ locator1: {To: []string{}, From: []string{"f1", "f2"}}}),
PullListMapEquals,
map[string]PullList{})
c.Check(
BuildPullLists(map[Locator]PullServers{
- locator1: PullServers{To: []string{"t1"}, From: []string{"f1", "f2"}}}),
+ locator1: {To: []string{"t1"}, From: []string{"f1", "f2"}}}),
PullListMapEquals,
map[string]PullList{
- "t1": PullList{PullRequest{locator1, []string{"f1", "f2"}}}})
+ "t1": {PullRequest{locator1, []string{"f1", "f2"}}}})
c.Check(
BuildPullLists(map[Locator]PullServers{
- locator1: PullServers{To: []string{"t1"}, From: []string{}}}),
+ locator1: {To: []string{"t1"}, From: []string{}}}),
PullListMapEquals,
- map[string]PullList{"t1": PullList{
+ map[string]PullList{"t1": {
PullRequest{locator1, []string{}}}})
c.Check(
BuildPullLists(map[Locator]PullServers{
- locator1: PullServers{
+ locator1: {
To: []string{"t1", "t2"},
From: []string{"f1", "f2"},
}}),
PullListMapEquals,
map[string]PullList{
- "t1": PullList{PullRequest{locator1, []string{"f1", "f2"}}},
- "t2": PullList{PullRequest{locator1, []string{"f1", "f2"}}},
+ "t1": {PullRequest{locator1, []string{"f1", "f2"}}},
+ "t2": {PullRequest{locator1, []string{"f1", "f2"}}},
})
locator2 := Locator{Digest: blockdigest.MakeTestBlockDigest(0xCabbed)}
c.Check(
BuildPullLists(map[Locator]PullServers{
- locator1: PullServers{To: []string{"t1"}, From: []string{"f1", "f2"}},
- locator2: PullServers{To: []string{"t2"}, From: []string{"f3", "f4"}}}),
+ locator1: {To: []string{"t1"}, From: []string{"f1", "f2"}},
+ locator2: {To: []string{"t2"}, From: []string{"f3", "f4"}}}),
PullListMapEquals,
map[string]PullList{
- "t1": PullList{PullRequest{locator1, []string{"f1", "f2"}}},
- "t2": PullList{PullRequest{locator2, []string{"f3", "f4"}}},
+ "t1": {PullRequest{locator1, []string{"f1", "f2"}}},
+ "t2": {PullRequest{locator2, []string{"f3", "f4"}}},
})
c.Check(
BuildPullLists(map[Locator]PullServers{
- locator1: PullServers{
+ locator1: {
To: []string{"t1"},
From: []string{"f1", "f2"}},
- locator2: PullServers{
+ locator2: {
To: []string{"t2", "t1"},
From: []string{"f3", "f4"}},
}),
PullListMapEquals,
map[string]PullList{
- "t1": PullList{
+ "t1": {
PullRequest{locator1, []string{"f1", "f2"}},
PullRequest{locator2, []string{"f3", "f4"}},
},
- "t2": PullList{
+ "t2": {
PullRequest{locator2, []string{"f3", "f4"}},
},
})
locator4 := Locator{Digest: blockdigest.MakeTestBlockDigest(0xFedBeef)}
c.Check(
BuildPullLists(map[Locator]PullServers{
- locator1: PullServers{
+ locator1: {
To: []string{"t1"},
From: []string{"f1", "f2"}},
- locator2: PullServers{
+ locator2: {
To: []string{"t2", "t1"},
From: []string{"f3", "f4"}},
- locator3: PullServers{
+ locator3: {
To: []string{"t3", "t2", "t1"},
From: []string{"f4", "f5"}},
- locator4: PullServers{
+ locator4: {
To: []string{"t4", "t3", "t2", "t1"},
From: []string{"f1", "f5"}},
}),
PullListMapEquals,
map[string]PullList{
- "t1": PullList{
+ "t1": {
PullRequest{locator1, []string{"f1", "f2"}},
PullRequest{locator2, []string{"f3", "f4"}},
PullRequest{locator3, []string{"f4", "f5"}},
PullRequest{locator4, []string{"f1", "f5"}},
},
- "t2": PullList{
+ "t2": {
PullRequest{locator2, []string{"f3", "f4"}},
PullRequest{locator3, []string{"f4", "f5"}},
PullRequest{locator4, []string{"f1", "f5"}},
},
- "t3": PullList{
+ "t3": {
PullRequest{locator3, []string{"f4", "f5"}},
PullRequest{locator4, []string{"f1", "f5"}},
},
- "t4": PullList{
+ "t4": {
PullRequest{locator4, []string{"f1", "f5"}},
},
})
}
func TestToCollectionIndexSet(t *testing.T) {
- VerifyToCollectionIndexSet(t, []int{6}, map[int][]int{6: []int{0}}, []int{0})
- VerifyToCollectionIndexSet(t, []int{4}, map[int][]int{4: []int{1}}, []int{1})
- VerifyToCollectionIndexSet(t, []int{4}, map[int][]int{4: []int{1, 9}}, []int{1, 9})
+ VerifyToCollectionIndexSet(t, []int{6}, map[int][]int{6: {0}}, []int{0})
+ VerifyToCollectionIndexSet(t, []int{4}, map[int][]int{4: {1}}, []int{1})
+ VerifyToCollectionIndexSet(t, []int{4}, map[int][]int{4: {1, 9}}, []int{1, 9})
VerifyToCollectionIndexSet(t, []int{5, 6},
- map[int][]int{5: []int{2, 3}, 6: []int{3, 4}},
+ map[int][]int{5: {2, 3}, 6: {3, 4}},
[]int{2, 3, 4})
VerifyToCollectionIndexSet(t, []int{5, 6},
- map[int][]int{5: []int{8}, 6: []int{4}},
+ map[int][]int{5: {8}, 6: {4}},
[]int{4, 8})
- VerifyToCollectionIndexSet(t, []int{6}, map[int][]int{5: []int{0}}, []int{})
+ VerifyToCollectionIndexSet(t, []int{6}, map[int][]int{5: {0}}, []int{})
}
func TestSimpleSummary(t *testing.T) {
rc := collection.MakeTestReadCollections([]collection.TestCollectionSpec{
- collection.TestCollectionSpec{ReplicationLevel: 1, Blocks: []int{1, 2}},
+ {ReplicationLevel: 1, Blocks: []int{1, 2}},
})
rc.Summarize(nil)
cIndex := rc.CollectionIndicesForTesting()
func TestMissingBlock(t *testing.T) {
rc := collection.MakeTestReadCollections([]collection.TestCollectionSpec{
- collection.TestCollectionSpec{ReplicationLevel: 1, Blocks: []int{1, 2}},
+ {ReplicationLevel: 1, Blocks: []int{1, 2}},
})
rc.Summarize(nil)
cIndex := rc.CollectionIndicesForTesting()
func TestUnderAndOverReplicatedBlocks(t *testing.T) {
rc := collection.MakeTestReadCollections([]collection.TestCollectionSpec{
- collection.TestCollectionSpec{ReplicationLevel: 2, Blocks: []int{1, 2}},
+ {ReplicationLevel: 2, Blocks: []int{1, 2}},
})
rc.Summarize(nil)
cIndex := rc.CollectionIndicesForTesting()
func TestMixedReplication(t *testing.T) {
rc := collection.MakeTestReadCollections([]collection.TestCollectionSpec{
- collection.TestCollectionSpec{ReplicationLevel: 1, Blocks: []int{1, 2}},
- collection.TestCollectionSpec{ReplicationLevel: 1, Blocks: []int{3, 4}},
- collection.TestCollectionSpec{ReplicationLevel: 2, Blocks: []int{5, 6}},
+ {ReplicationLevel: 1, Blocks: []int{1, 2}},
+ {ReplicationLevel: 1, Blocks: []int{3, 4}},
+ {ReplicationLevel: 2, Blocks: []int{5, 6}},
})
rc.Summarize(nil)
cIndex := rc.CollectionIndicesForTesting()
ttl := int64(_ttl.(float64))
// expire unreferenced blocks more than "ttl" seconds old.
- expiry := time.Now().UTC().Unix() - ttl
+ expiry := time.Now().UTC().UnixNano() - ttl*1e9
return buildTrashListsInternal(writableServers, keepServerInfo, expiry, keepBlocksNotInCollections), nil
}
var keepServerInfo = keep.ReadServers{
KeepServerIndexToAddress: []keep.ServerAddress{sv0, sv1},
BlockToServers: map[blockdigest.DigestWithSize][]keep.BlockServerInfo{
- block0: []keep.BlockServerInfo{
- keep.BlockServerInfo{0, 99},
- keep.BlockServerInfo{1, 101}},
- block1: []keep.BlockServerInfo{
- keep.BlockServerInfo{0, 99},
- keep.BlockServerInfo{1, 101}}}}
+ block0: {
+ {0, 99},
+ {1, 101}},
+ block1: {
+ {0, 99},
+ {1, 101}}}}
// only block0 is in delete set
var bs = make(BlockSet)
// Test trash list where only sv0 is on writable list.
c.Check(buildTrashListsInternal(
map[string]struct{}{
- sv0.URL(): struct{}{}},
+ sv0.URL(): {}},
&keepServerInfo,
110,
bs),
DeepEquals,
map[string]keep.TrashList{
- "http://keep0.example.com:80": keep.TrashList{keep.TrashRequest{"000000000000000000000000deadbeef", 99}}})
+ "http://keep0.example.com:80": {keep.TrashRequest{"000000000000000000000000deadbeef", 99}}})
// Test trash list where both sv0 and sv1 are on writable list.
c.Check(buildTrashListsInternal(
map[string]struct{}{
- sv0.URL(): struct{}{},
- sv1.URL(): struct{}{}},
+ sv0.URL(): {},
+ sv1.URL(): {}},
&keepServerInfo,
110,
bs),
DeepEquals,
map[string]keep.TrashList{
- "http://keep0.example.com:80": keep.TrashList{keep.TrashRequest{"000000000000000000000000deadbeef", 99}},
- "http://keep1.example.com:80": keep.TrashList{keep.TrashRequest{"000000000000000000000000deadbeef", 101}}})
+ "http://keep0.example.com:80": {keep.TrashRequest{"000000000000000000000000deadbeef", 99}},
+ "http://keep1.example.com:80": {keep.TrashRequest{"000000000000000000000000deadbeef", 101}}})
// Test trash list where only block on sv0 is expired
c.Check(buildTrashListsInternal(
map[string]struct{}{
- sv0.URL(): struct{}{},
- sv1.URL(): struct{}{}},
+ sv0.URL(): {},
+ sv1.URL(): {}},
&keepServerInfo,
100,
bs),
DeepEquals,
map[string]keep.TrashList{
- "http://keep0.example.com:80": keep.TrashList{keep.TrashRequest{"000000000000000000000000deadbeef", 99}}})
+ "http://keep0.example.com:80": {keep.TrashRequest{"000000000000000000000000deadbeef", 99}}})
}
return err
}
bal.DefaultReplication = dd.DefaultCollectionReplication
- bal.MinMtime = time.Now().Unix() - dd.BlobSignatureTTL
+ bal.MinMtime = time.Now().UnixNano() - dd.BlobSignatureTTL*1e9
errs := make(chan error, 2+len(bal.KeepServices))
wg := sync.WaitGroup{}
}(srv)
}
var lastErr error
- for _ = range bal.KeepServices {
+ for range bal.KeepServices {
if err := <-errs; err != nil {
bal.logf("%v", err)
lastErr = err
bal.KeepServices[srv.UUID] = srv
}
- bal.MinMtime = time.Now().Unix() - bal.signatureTTL
+ bal.MinMtime = time.Now().UnixNano() - bal.signatureTTL*1e9
}
func (bal *balancerSuite) TestPerfect(c *check.C) {
// replList is like srvList but returns an "existing replicas" slice,
// suitable for a BlockState test fixture.
func (bal *balancerSuite) replList(knownBlockID int, order slots) (repls []Replica) {
- mtime := time.Now().Unix() - bal.signatureTTL - 86400
+ mtime := time.Now().UnixNano() - (bal.signatureTTL+86400)*1e9
for _, srv := range bal.srvList(knownBlockID, order) {
repls = append(repls, Replica{srv, mtime})
mtime++
// Trashed blob; exclude it from response
continue
}
- fmt.Fprintf(writer, "%s+%d %d\n", b.Name, b.Properties.ContentLength, t.Unix())
+ fmt.Fprintf(writer, "%s+%d %d\n", b.Name, b.Properties.ContentLength, t.UnixNano())
}
if resp.NextMarker == "" {
return nil
flag.IntVar(
&permissionTTLSec,
"blob-signature-ttl",
- int(time.Duration(2*7*24*time.Hour).Seconds()),
- "Lifetime of blob permission signatures. Modifying the ttl will invalidate all existing signatures. "+
+ 2*7*24*3600,
+ "Lifetime of blob permission signatures in seconds. Modifying the ttl will invalidate all existing signatures. "+
"See services/api/config/application.default.yml.")
flag.BoolVar(
&flagSerializeIO,
}
func s3regions() (okList []string) {
- for r, _ := range aws.Regions {
+ for r := range aws.Regions {
okList = append(okList, r)
}
return
if !v.isKeepBlock(key.Key) {
continue
}
- fmt.Fprintf(writer, "%s+%d %d\n", key.Key, key.Size, t.Unix())
+ fmt.Fprintf(writer, "%s+%d %d\n", key.Key, key.Size, t.UnixNano())
}
if !listResp.IsTruncated {
break
// TrashItem deletes the indicated block from every writable volume.
func TrashItem(trashRequest TrashRequest) {
- reqMtime := time.Unix(trashRequest.BlockMtime, 0)
+ reqMtime := time.Unix(0, trashRequest.BlockMtime)
if time.Since(reqMtime) < blobSignatureTTL {
log.Printf("WARNING: data manager asked to delete a %v old block %v (BlockMtime %d = %v), but my blobSignatureTTL is %v! Skipping.",
time.Since(reqMtime),
log.Printf("%v Delete(%v): %v", volume, trashRequest.Locator, err)
continue
}
- if trashRequest.BlockMtime != mtime.Unix() {
- log.Printf("%v Delete(%v): mtime on volume is %v does not match trash list value %v", volume, trashRequest.Locator, mtime.Unix(), trashRequest.BlockMtime)
+ if trashRequest.BlockMtime != mtime.UnixNano() {
+ log.Printf("%v Delete(%v): stored mtime %v does not match trash list value %v", volume, trashRequest.Locator, mtime.UnixNano(), trashRequest.BlockMtime)
continue
}
// Create TrashRequest for the test
trashRequest := TrashRequest{
Locator: testData.DeleteLocator,
- BlockMtime: oldBlockTime.Unix(),
+ BlockMtime: oldBlockTime.UnixNano(),
}
// Run trash worker and put the trashRequest on trashq
"os"
"regexp"
"sort"
+ "strconv"
"strings"
"time"
v := factory(t)
defer v.Teardown()
+ // minMtime and maxMtime are the minimum and maximum
+ // acceptable values the index can report for our test
+ // blocks. 1-second precision is acceptable.
+ minMtime := time.Now().UTC().UnixNano()
+ minMtime -= minMtime % 1e9
+
v.PutRaw(TestHash, TestBlock)
v.PutRaw(TestHash2, TestBlock2)
v.PutRaw(TestHash3, TestBlock3)
+ maxMtime := time.Now().UTC().UnixNano()
+ if maxMtime%1e9 > 0 {
+ maxMtime -= maxMtime % 1e9
+ maxMtime += 1e9
+ }
+
// Blocks whose names aren't Keep hashes should be omitted from
// index
v.PutRaw("fffffffffnotreallyahashfffffffff", nil)
indexRows := strings.Split(string(buf.Bytes()), "\n")
sort.Strings(indexRows)
sortedIndex := strings.Join(indexRows, "\n")
- m, err := regexp.MatchString(
- `^\n`+TestHash+`\+\d+ \d+\n`+
- TestHash3+`\+\d+ \d+\n`+
- TestHash2+`\+\d+ \d+$`,
- sortedIndex)
- if err != nil {
- t.Error(err)
- } else if !m {
+ m := regexp.MustCompile(
+ `^\n` + TestHash + `\+\d+ (\d+)\n` +
+ TestHash3 + `\+\d+ \d+\n` +
+ TestHash2 + `\+\d+ \d+$`,
+ ).FindStringSubmatch(sortedIndex)
+ if m == nil {
t.Errorf("Got index %q for empty prefix", sortedIndex)
+ } else {
+ mtime, err := strconv.ParseInt(m[1], 10, 64)
+ if err != nil {
+ t.Error(err)
+ } else if mtime < minMtime || mtime > maxMtime {
+ t.Errorf("got %d for TestHash timestamp, expected %d <= t <= %d",
+ mtime, minMtime, maxMtime)
+ }
}
for _, prefix := range []string{"f", "f15", "f15ac"} {
for _, prefix := range []string{"zero", "zip", "zilch"} {
buf = new(bytes.Buffer)
- v.IndexTo(prefix, buf)
+ err := v.IndexTo(prefix, buf)
if err != nil {
t.Errorf("Got error on IndexTo with no such prefix %v", err.Error())
} else if buf.Len() != 0 {
return e
}
defer unlockfile(f)
- now := time.Now().Unix()
- utime := syscall.Utimbuf{now, now}
- return syscall.Utime(p, &utime)
+ ts := syscall.NsecToTimespec(time.Now().UnixNano())
+ return syscall.UtimesNano(p, []syscall.Timespec{ts, ts})
}
// Mtime returns the stored timestamp for the given locator.
_, err = fmt.Fprint(w,
name,
"+", fileInfo[0].Size(),
- " ", fileInfo[0].ModTime().Unix(),
+ " ", fileInfo[0].ModTime().UnixNano(),
"\n")
}
blockdir.Close()
gate := make(chan struct{})
go func() {
<-gate
- for _ = range b.NextItem {
+ for range b.NextItem {
<-gate
time.Sleep(time.Millisecond)
b.DoneItem <- struct{}{}
projects / arvados.git / commitdiff