end
end
if resource_class == Collection
- dst.manifest_text = Collection.select([:manifest_text]).where(uuid: src.uuid).first.manifest_text
+ dst.manifest_text = Collection.select([:manifest_text]).where(uuid: src.uuid).with_count("none").first.manifest_text
end
when :move
dst = src
uuids, source_paths = selected_collection_files params
new_coll = Arv::Collection.new
- Collection.where(uuid: uuids.uniq).
+ Collection.where(uuid: uuids.uniq).with_count("none").
select([:uuid, :manifest_text]).each do |coll|
src_coll = Arv::Collection.new(coll.manifest_text)
src_pathlist = source_paths[coll.uuid]
def is_starred
links = Link.where(tail_uuid: current_user.uuid,
head_uuid: @object.uuid,
- link_class: 'star')
+ link_class: 'star').with_count("none")
return links.andand.any?
end
@@notification_tests.push lambda { |controller, current_user|
return nil if Rails.configuration.Services.WebShell.ExternalURL != URI("")
- AuthorizedKey.limit(1).where(authorized_user_uuid: current_user.uuid).each do
+ AuthorizedKey.limit(1).with_count('none').where(authorized_user_uuid: current_user.uuid).each do
return nil
end
return lambda { |view|
}
@@notification_tests.push lambda { |controller, current_user|
- Collection.limit(1).where(created_by: current_user.uuid).each do
+ Collection.limit(1).with_count('none').where(created_by: current_user.uuid).each do
return nil
end
return lambda { |view|
@@notification_tests.push lambda { |controller, current_user|
if PipelineInstance.api_exists?(:index)
- PipelineInstance.limit(1).where(created_by: current_user.uuid).each do
+ PipelineInstance.limit(1).with_count('none').where(created_by: current_user.uuid).each do
return nil
end
else
helper_method :recent_jobs_and_pipelines
def recent_jobs_and_pipelines
(Job.limit(10) |
- PipelineInstance.limit(10)).
+ PipelineInstance.limit(10).with_count("none")).
sort_by do |x|
(x.finished_at || x.started_at rescue nil) || x.modified_at || x.created_at
end.reverse
helper_method :running_pipelines
def running_pipelines
- pi = PipelineInstance.order(["started_at asc", "created_at asc"]).filter([["state", "in", ["RunningOnServer", "RunningOnClient"]]])
+ pi = PipelineInstance.order(["started_at asc", "created_at asc"]).with_count("none").filter([["state", "in", ["RunningOnServer", "RunningOnClient"]]])
jobs = {}
pi.each do |pl|
pl.components.each do |k,v|
end
if jobs.keys.any?
- Job.filter([["uuid", "in", jobs.keys]]).each do |j|
+ Job.filter([["uuid", "in", jobs.keys]]).with_count("none").each do |j|
jobs[j[:uuid]] = j
end
procs = {}
if PipelineInstance.api_exists?(:index)
cols = %w(uuid owner_uuid created_at modified_at pipeline_template_uuid name state started_at finished_at)
- pipelines = PipelineInstance.select(cols).limit(lim).order(["created_at desc"])
+ pipelines = PipelineInstance.select(cols).limit(lim).order(["created_at desc"]).with_count("none")
pipelines.results.each { |pi| procs[pi] = pi.created_at }
end
- crs = ContainerRequest.limit(lim).order(["created_at desc"]).filter([["requesting_container_uuid", "=", nil]])
+ crs = ContainerRequest.limit(lim).with_count("none").order(["created_at desc"]).filter([["requesting_container_uuid", "=", nil]])
crs.results.each { |c| procs[c] = c.created_at }
Hash[procs.sort_by {|key, value| value}].keys.reverse.first(lim)
helper_method :recent_collections
def recent_collections lim
- c = Collection.limit(lim).order(["modified_at desc"]).results
+ c = Collection.limit(lim).with_count("none").order(["modified_at desc"]).results
own = {}
- Group.filter([["uuid", "in", c.map(&:owner_uuid)]]).each do |g|
+ Group.filter([["uuid", "in", c.map(&:owner_uuid)]]).with_count("none").each do |g|
own[g[:uuid]] = g
end
{collections: c, owners: own}
return if defined?(@starred_projects) && @starred_projects
links = Link.filter([['tail_uuid', '=', user.uuid],
['link_class', '=', 'star'],
- ['head_uuid', 'is_a', 'arvados#group']]).select(%w(head_uuid))
+ ['head_uuid', 'is_a', 'arvados#group']]).with_count("none").select(%w(head_uuid))
uuids = links.collect { |x| x.head_uuid }
- starred_projects = Group.filter([['uuid', 'in', uuids]]).order('name')
+ starred_projects = Group.filter([['uuid', 'in', uuids]]).order('name').with_count("none")
@starred_projects = starred_projects.results
end
end
# TODO: make sure we get every page of results from API server
- Link.filter([['head_uuid', 'in', uuids]]).each do |link|
+ Link.filter([['head_uuid', 'in', uuids]]).with_count("none").each do |link|
@all_links_for[link.head_uuid] << link
end
@all_links_for
end
# TODO: make sure we get every page of results from API server
- Collection.where(uuid: uuids).each do |collection|
+ Collection.where(uuid: uuids).with_count("none").each do |collection|
@all_collections_for[collection.uuid] << collection
end
@all_collections_for
end
# TODO: make sure we get every page of results from API server
- Collection.where(uuid: uuids).each do |collection|
+ Collection.where(uuid: uuids).with_count("none").each do |collection|
@all_log_collections_for[collection.uuid] << collection
end
@all_log_collections_for
@all_pdhs_for[x] = []
end
- Collection.select(%w(portable_data_hash)).where(portable_data_hash: pdhs).distinct().each do |collection|
+ Collection.select(%w(portable_data_hash)).where(portable_data_hash: pdhs).distinct().with_count("none").each do |collection|
@all_pdhs_for[collection.portable_data_hash] << collection
end
@all_pdhs_for
end
unless link_uuids.empty?
- Link.select([:head_uuid]).where(uuid: link_uuids).each do |link|
+ Link.select([:head_uuid]).where(uuid: link_uuids).with_count("none").each do |link|
if ArvadosBase::resource_class_for_uuid(link.head_uuid) == Collection
coll_ids << link.head_uuid
end
end
unless pdhs.empty?
- Collection.where(portable_data_hash: pdhs.uniq).
+ Collection.where(portable_data_hash: pdhs.uniq).with_count("none").
select([:uuid, :portable_data_hash]).each do |coll|
unless source_paths[coll.portable_data_hash].empty?
uuids << coll.uuid
['link_class', '=', 'resources'],
['name', '=', 'wants'],
['tail_uuid', '=', current_user.uuid],
- ['head_uuid', '=', @object.uuid]])
+ ['head_uuid', '=', @object.uuid]]).with_count("none")
logger.debug persist_links.inspect
else
return unprocessable "Invalid value #{value.inspect}"
@select ||= Collection.columns.map(&:name)
base_search = Collection.select(@select)
if params[:search].andand.length.andand > 0
- tags = Link.where(any: ['contains', params[:search]])
+ tags = Link.where(any: ['contains', params[:search]]).with_count("none")
@objects = (base_search.where(uuid: tags.collect(&:head_uuid)) |
base_search.where(any: ['contains', params[:search]])).
uniq { |c| c.uuid }
@objects = base_search.limit(limit).offset(offset)
end
- @links = Link.where(head_uuid: @objects.collect(&:uuid))
+ @links = Link.where(head_uuid: @objects.collect(&:uuid)).with_count("none")
@collection_info = {}
@objects.each do |c|
@collection_info[c.uuid] = {
else
if Job.api_exists?(:index)
jobs_with = lambda do |conds|
- Job.limit(RELATION_LIMIT).where(conds)
+ Job.limit(RELATION_LIMIT).with_count("none").where(conds)
.results.sort_by { |j| j.finished_at || j.created_at }
end
@output_of = jobs_with.call(output: @object.portable_data_hash)
@log_of = jobs_with.call(log: @object.portable_data_hash)
end
- @project_links = Link.limit(RELATION_LIMIT).order("modified_at DESC")
+ @project_links = Link.limit(RELATION_LIMIT).with_count("none").order("modified_at DESC")
.where(head_uuid: @object.uuid, link_class: 'name').results
- project_hash = Group.where(uuid: @project_links.map(&:tail_uuid)).to_hash
+ project_hash = Group.where(uuid: @project_links.map(&:tail_uuid)).with_count("none").to_hash
@projects = project_hash.values
- @permissions = Link.limit(RELATION_LIMIT).order("modified_at DESC")
+ @permissions = Link.limit(RELATION_LIMIT).with_count("none").order("modified_at DESC")
.where(head_uuid: @object.uuid, link_class: 'permission',
name: 'can_read').results
@search_sharing = search_scopes
# Search for child CRs
if cr[:container_uuid]
- child_crs = ContainerRequest.where(requesting_container_uuid: cr[:container_uuid])
+ child_crs = ContainerRequest.where(requesting_container_uuid: cr[:container_uuid]).with_count("none")
child_crs.each do |child|
nodes[child[:uuid]] = child
# Batch requests to get all related collections
# First fetch output collections by UUID.
- Collection.filter([['uuid', 'in', col_uuids.uniq]]).each do |c|
+ Collection.filter([['uuid', 'in', col_uuids.uniq]]).with_count("none").each do |c|
output_pdhs << c[:portable_data_hash]
pdh_to_col[c[:portable_data_hash]] = c
nodes[c[:uuid]] = c
end
# Next, get input collections by PDH.
Collection.filter(
- [['portable_data_hash', 'in', col_pdhs - output_pdhs]]).each do |c|
+ [['portable_data_hash', 'in', col_pdhs - output_pdhs]]).with_count("none").each do |c|
nodes[c[:portable_data_hash]] = c
end
def cancel
if @object.container_uuid
- c = Container.select(['state']).where(uuid: @object.container_uuid).first
+ c = Container.select(['state']).where(uuid: @object.container_uuid).with_count("none").first
if c && c.state != 'Running'
# If the container hasn't started yet, setting priority=0
# leaves our request in "Committed" state and doesn't cancel
def index
@groups = Group.filter [['group_class', '!=', 'project']]
@group_uuids = @groups.collect &:uuid
- @links_from = Link.where link_class: 'permission', tail_uuid: @group_uuids
- @links_to = Link.where link_class: 'permission', head_uuid: @group_uuids
+ @links_from = Link.where(link_class: 'permission', tail_uuid: @group_uuids).with_count("none")
+ @links_to = Link.where(link_class: 'permission', head_uuid: @group_uuids).with_count("none")
render_index
end
nodes[j[:script_version]] = {:uuid => j[:script_version]}
end
- Collection.where(uuid: collections).each do |c|
+ Collection.where(uuid: collections).with_count("none").each do |c|
nodes[c[:portable_data_hash]] = c
end
- Collection.where(portable_data_hash: hashes).each do |c|
+ Collection.where(portable_data_hash: hashes).with_count("none").each do |c|
nodes[c[:portable_data_hash]] = c
end
jobs = jobs.compact.uniq
if jobs.any?
- Job.where(uuid: jobs).each do |j|
+ Job.where(uuid: jobs).with_count("none").each do |j|
job_uuid = j.uuid
provenance[job_uuid] = j
hashes = hashes.compact.uniq
if hashes.any?
- Collection.where(portable_data_hash: hashes).each do |c|
+ Collection.where(portable_data_hash: hashes).with_count("none").each do |c|
hash_uuid = c.portable_data_hash
provenance[hash_uuid] = c
pips[hash_uuid] = 0 unless pips[hash_uuid] != nil
collections = collections.compact.uniq
if collections.any?
- Collection.where(uuid: collections).each do |c|
+ Collection.where(uuid: collections).with_count("none").each do |c|
collection_uuid = c.uuid
provenance[collection_uuid] = c
pips[collection_uuid] = 0 unless pips[collection_uuid] != nil
def destroy
while (objects = Link.filter([['owner_uuid','=',@object.uuid],
- ['tail_uuid','=',@object.uuid]])).any?
+ ['tail_uuid','=',@object.uuid]]).with_count("none")).any?
objects.each do |object|
object.destroy
end
if !owner_filter.andand.any?
filters = @filters + [["owner_uuid", "=", current_user.uuid]]
- my_repos = Repository.all.order("name ASC").limit(limit).offset(offset).filter(filters).results
+ my_repos = Repository.all.order("name ASC").limit(limit).with_count("none").offset(offset).filter(filters).results
else # done fetching all owned repositories
my_repos = []
end
end
filters = @filters + [["owner_uuid", "!=", current_user.uuid]]
- other_repos = Repository.all.order("name ASC").limit(limit).offset(offset).filter(filters).results
+ other_repos = Repository.all.order("name ASC").limit(limit).with_count("none").offset(offset).filter(filters).results
@objects = (my_repos + other_repos).first(limit)
end
base_search = base_search.filter([["modified_at", "<=", last_mod_at], ["uuid", "not in", last_uuids]])
end
- base_search = base_search.include_trash(true).limit(limit).offset(offset)
+ base_search = base_search.include_trash(true).limit(limit).with_count("none").offset(offset)
if params[:filters].andand.length.andand > 0
- tags = Link.filter(params[:filters])
+ tags = Link.filter(params[:filters]).with_count("none")
tagged = []
if tags.results.length > 0
tagged = query_on.include_trash(true).where(uuid: tags.collect(&:head_uuid))
owner_uuids = @objects.collect(&:owner_uuid).uniq
@owners = {}
@not_trashed = {}
- Group.filter([["uuid", "in", owner_uuids]]).include_trash(true).each do |grp|
+ Group.filter([["uuid", "in", owner_uuids]]).with_count("none").include_trash(true).each do |grp|
@owners[grp.uuid] = grp
end
- User.filter([["uuid", "in", owner_uuids]]).include_trash(true).each do |grp|
+ User.filter([["uuid", "in", owner_uuids]]).with_count("none").include_trash(true).each do |grp|
@owners[grp.uuid] = grp
@not_trashed[grp.uuid] = true
end
- Group.filter([["uuid", "in", owner_uuids]]).select([:uuid]).each do |grp|
+ Group.filter([["uuid", "in", owner_uuids]]).with_count("none").select([:uuid]).each do |grp|
@not_trashed[grp.uuid] = true
end
else
def activity
@breadcrumb_page_name = nil
- @users = User.limit(params[:limit])
+ @users = User.limit(params[:limit]).with_count("none")
@user_activity = {}
@activity = {
logins: {},
filter([[:event_type, '=', 'login'],
[:object_kind, '=', 'arvados#user'],
[:created_at, '>=', threshold_start],
- [:created_at, '<', threshold_end]])
+ [:created_at, '<', threshold_end]]).with_count("none")
@activity[:jobs][span] = Job.select(%w(uuid modified_by_user_uuid)).
filter([[:created_at, '>=', threshold_start],
- [:created_at, '<', threshold_end]])
+ [:created_at, '<', threshold_end]]).with_count("none")
@activity[:pipeline_instances][span] = PipelineInstance.select(%w(uuid modified_by_user_uuid)).
filter([[:created_at, '>=', threshold_start],
- [:created_at, '<', threshold_end]])
+ [:created_at, '<', threshold_end]]).with_count("none")
@activity.each do |type, act|
records = act[span]
@users.each do |u|
def storage
@breadcrumb_page_name = nil
- @users = User.limit(params[:limit])
+ @users = User.limit(params[:limit]).with_count("none")
@user_storage = {}
total_storage = {}
@log_date = {}
@my_jobs = Job.
limit(10).
order('created_at desc').
+ with_count('none').
where(created_by: current_user.uuid)
@my_collections = Collection.
limit(10).
order('created_at desc').
+ with_count('none').
where(created_by: current_user.uuid)
collection_uuids = @my_collections.collect &:uuid
end
Link.filter([['head_uuid', 'in', collection_uuids],
- ['link_class', 'in', ['tag', 'resources']]]).
+ ['link_class', 'in', ['tag', 'resources']]]).with_count("none")
each do |link|
case link.link_class
when 'tag'
@my_pipelines = PipelineInstance.
limit(10).
order('created_at desc').
+ with_count('none').
where(created_by: current_user.uuid)
respond_to do |f|
@my_vm_logins = {}
Link.where(tail_uuid: @object.uuid,
link_class: 'permission',
- name: 'can_login').
+ name: 'can_login').with_count("none").
each do |perm_link|
if perm_link.properties.andand[:username]
@my_vm_logins[perm_link.head_uuid] ||= []
@my_vm_logins[perm_link.head_uuid] << perm_link.properties[:username]
end
end
- @my_virtual_machines = VirtualMachine.where(uuid: @my_vm_logins.keys)
+ @my_virtual_machines = VirtualMachine.where(uuid: @my_vm_logins.keys).with_count("none")
end
def ssh_keys
oid_login_perms = Link.where(tail_uuid: user.email,
head_kind: 'arvados#user',
link_class: 'permission',
- name: 'can_login')
+ name: 'can_login').with_count("none")
if oid_login_perms.any?
prefix_properties = oid_login_perms.first.properties
repo_perms = Link.where(tail_uuid: user.uuid,
head_kind: 'arvados#repository',
link_class: 'permission',
- name: 'can_write')
+ name: 'can_write').with_count("none")
if repo_perms.any?
repo_uuid = repo_perms.first.head_uuid
- repos = Repository.where(head_uuid: repo_uuid)
+ repos = Repository.where(head_uuid: repo_uuid).with_count("none")
if repos.any?
repo_name = repos.first.name
current_selections[:repo_name] = repo_name
vm_login_perms = Link.where(tail_uuid: user.uuid,
head_kind: 'arvados#virtualMachine',
link_class: 'permission',
- name: 'can_login')
+ name: 'can_login').with_count("none")
if vm_login_perms.any?
vm_perm = vm_login_perms.first
vm_uuid = vm_perm.head_uuid
Link.where(tail_uuid: current_user.uuid,
head_uuid: @objects.collect(&:uuid),
link_class: 'permission',
- name: 'can_login').
+ name: 'can_login').with_count("none").
each do |perm_link|
if perm_link.properties.andand[:username]
@vm_logins[perm_link.head_uuid] ||= []
# get next page of pipeline_templates
if PipelineTemplate.api_exists?(:index)
filters = @filters + [["uuid", "is_a", ["arvados#pipelineTemplate"]]]
- pipelines = PipelineTemplate.limit(@limit).order(["created_at desc"]).filter(filters)
+ pipelines = PipelineTemplate.limit(@limit).with_count("none").order(["created_at desc"]).filter(filters)
end
# get next page of workflows
filters = @filters + [["uuid", "is_a", ["arvados#workflow"]]]
- workflows = Workflow.limit(@limit).order(["created_at desc"]).filter(filters)
+ workflows = Workflow.limit(@limit).order(["created_at desc"]).with_count("none").filter(filters)
@objects = (pipelines.to_a + workflows.to_a).sort_by(&:created_at).reverse.first(@limit)
# get next page of pipeline_instances
if PipelineInstance.api_exists?(:index)
filters = @filters + [["uuid", "is_a", ["arvados#pipelineInstance"]]]
- pipelines = PipelineInstance.limit(@limit).order(["created_at desc"]).filter(filters)
+ pipelines = PipelineInstance.limit(@limit).order(["created_at desc"]).filter(filters).with_count("none")
end
if params[:show_children]
# get next page of jobs
if Job.api_exists?(:index)
filters = @filters + [["uuid", "is_a", ["arvados#job"]]]
- jobs = Job.limit(@limit).order(["created_at desc"]).filter(filters)
+ jobs = Job.limit(@limit).order(["created_at desc"]).filter(filters).with_count("none")
end
end
if !params[:show_children]
filters << ["requesting_container_uuid", "=", nil]
end
- crs = ContainerRequest.limit(@limit).order(["created_at desc"]).filter(filters)
+ crs = ContainerRequest.limit(@limit).order(["created_at desc"]).filter(filters).with_count("none")
@objects = (jobs.to_a + pipelines.to_a + crs.to_a).sort_by(&:created_at).reverse.first(@limit)
if @objects.any?
c[:job][:uuid] if c.is_a?(Hash) and c[:job].is_a?(Hash)
}.compact
job = {}
- Job.where(uuid: jobuuids).each do |j|
+ Job.where(uuid: jobuuids).with_count("none").each do |j|
job[j[:uuid]] = j
end
container_uuid = if @proxied.is_a?(Container) then uuid else get(:container_uuid) end
if container_uuid
cols = ContainerRequest.columns.map(&:name) - %w(id updated_at mounts secret_mounts runtime_token)
- my_children = @child_proxies || ContainerRequest.select(cols).where(requesting_container_uuid: container_uuid).results if !my_children
+ my_children = @child_proxies || ContainerRequest.select(cols).where(requesting_container_uuid: container_uuid).with_count("none").results if !my_children
my_child_containers = my_children.map(&:container_uuid).compact.uniq
grandchildren = {}
my_child_containers.each { |c| grandchildren[c] = []} if my_child_containers.any?
- reqs = ContainerRequest.select(cols).where(requesting_container_uuid: my_child_containers).results if my_child_containers.any?
+ reqs = ContainerRequest.select(cols).where(requesting_container_uuid: my_child_containers).with_count("none").results if my_child_containers.any?
reqs.each {|cr| grandchildren[cr.requesting_container_uuid] << cr} if reqs
my_children.each do |cr|
Keep::Locator.parse(loc_s)
end
if log_pdhs.any? and
- Collection.where(portable_data_hash: log_pdhs).limit(1).results.any?
+ Collection.where(portable_data_hash: log_pdhs).limit(1).with_count("none").results.any?
true
elsif log_uuids.any? and
- Collection.where(uuid: log_uuids).limit(1).results.any?
+ Collection.where(uuid: log_uuids).limit(1).with_count("none").results.any?
true
else
stderr_log_query(1).results.any?
items = []
jobs = {}
- results = Job.where(uuid: @proxied.job_ids.values).results
+ results = Job.where(uuid: @proxied.job_ids.values).with_count("none").results
results.each do |j|
jobs[j.uuid] = j
end
Currently only reads @RemoteClusters@ from centralized configuration. Still requires component-specific configuration file.
+h2(#keepproxy). keepproxy
+
+The legacy keepproxy config (loaded from @/etc/arvados/keepproxy/keepproxy.yml@ or a different location specified via -legacy-keepproxy-config command line argument) takes precedence over the centralized config. After you migrate everything from the legacy config to the centralized config, you should delete @/etc/arvados/keepproxy/keepproxy.yml@ and stop using the -legacy-keepproxy-config argument.
+
+h2(#arv-git-httpd). arv-git-httpd
+
+The legacy arv-git-httpd config (loaded from @/etc/arvados/git-httpd/git-httpd.yml@ or a different location specified via -legacy-git-httpd-config command line argument) takes precedence over the centralized config. After you migrate everything from the legacy config to the centralized config, you should delete @/etc/arvados/git-httpd/git-httpd.yml@ and stop using the -legacy-git-httpd-config argument.
+
+
h2. arvados-controller
Already uses centralized config exclusively. No migration needed.
h3(#master). development master (as of 2019-08-12)
+h4. Arv-git-httpd configuration migration
+
+(feature "#14712":https://dev.arvados.org/issues/14712 ) The arv-git-httpd package can now be configured using the centralized configuration file at @/etc/arvados/config.yml@. Configuration via individual command line arguments is no longer available. Please see "arv-git-httpd's config migration guide":{{site.baseurl}}/admin/config-migration.html#arv-git-httpd for more details.
+
h4. Keep-web dropped support on command line flags configuration
As we're migrating to a central cluster configuration file, the already deprecated way of getting configurations via environment variables and command line flags isn't valid anymore. Current keep-web supports both the now legacy @keep-web.yml@ config format (used by Arvados 1.4) and the new cluster config file format. Please check "keep-web's install guide":{{site.baseurl}}/install/install-keep-web.html for more details.
So that older Arvados sites don't lose access to legacy records, the API has been converted to read-only. Creating and updating jobs (and related types job_task, pipeline_template and pipeline_instance) is disabled and much of the business logic related has been removed, along with various other code specific to the jobs API. Specifically, the following programs associated with the jobs API have been removed: @crunch-dispatch.rb@, @crunch-job@, @crunchrunner@, @arv-run-pipeline-instance@, @arv-run@.
+h4. Keepproxy configuration migration
+
+(feature "#14715":https://dev.arvados.org/issues/14715 ) Keepproxy can now be configured using the centralized config at @/etc/arvados/config.yml@. Configuration via individual command line arguments is no longer available and the @DisableGet@, @DisablePut@, and @PIDFile@ configuration options are no longer supported. If you are still using the legacy config and @DisableGet@ or @DisablePut@ are set to true or @PIDFile@ has a value, keepproxy will produce an error and fail to start. Please see "keepproxy's config migration guide":{{site.baseurl}}/admin/config-migration.html#keepproxy for more details.
+
h4. No longer stripping ':' from strings in serialized database columns
(bug "#15311":https://dev.arvados.org/issues/15311 ) Strings read from serialized columns in the database with a leading ':' would have the ':' stripped after loading the record. This behavior existed due to legacy serialization behavior which stored Ruby symbols with a leading ':'. Unfortunately this corrupted fields where the leading ":" was intentional. This behavior has been removed.
h3. Configure the API server to advertise the correct SSH URLs
-In your API server's @application.yml@ file, add the following entry:
+Edit the cluster config at @/etc/arvados/config.yml@ and set @Services.GitSSH.ExternalURL@. Replace @uuid_prefix@ with your cluster id.
<notextile>
-<pre><code>git_repo_ssh_base: "git@git.<span class="userinput">uuid_prefix.your.domain</span>:"
+<pre><code>Clusters:
+ <span class="userinput">uuid_prefix</span>:
+ Services:
+ GitSSH:
+ ExternalURL: <span class="userinput">git@git.uuid_prefix.your.domain:</span>
</code></pre>
</notextile>
{% include 'notebox_end' %}
-Create the configuration file @/etc/arvados/git-httpd/git-httpd.yml@. Run @arvados-git-httpd -h@ to learn more about configuration entries.
+Edit the cluster config at @/etc/arvados/config.yml@ and set the following values. Replace @uuid_prefix@ with your cluster id.
<notextile>
-<pre><code>Client:
- APIHost: <b>uuid_prefix.your.domain</b>
- Insecure: false
-GitCommand: /var/lib/arvados/git/gitolite/src/gitolite-shell
-GitoliteHome: /var/lib/arvados/git
-Listen: :9001
-RepoRoot: /var/lib/arvados/git/repositories
+<pre><code>Clusters:
+ <span class="userinput">uuid_prefix</span>:
+ Services:
+ GitHTTP:
+ ExternalURL: <span class="userinput">https://git.uuid_prefix.your.domain/</span>
+ InternalURLs:
+ <span class="userinput">"http://localhost:9001": {}</span>
+ Git:
+ GitCommand: <span class="userinput">/var/lib/arvados/git/gitolite/src/gitolite-shell</span>
+ GitoliteHome: <span class="userinput">/var/lib/arvados/git</span>
+ Repositories: <span class="userinput">/var/lib/arvados/git/repositories</span>
</code></pre>
</notextile>
+Make sure to include the trailing slash for @Services.GitHTTP.ExternalURL@.
+
Restart the systemd service to ensure the new configuration is used.
+
<notextile>
<pre><code>~$ <span class="userinput">sudo systemctl restart arvados-git-httpd</span>
</code></pre>
</code></pre>
</notextile>
-h3. Configure the API server to advertise the correct HTTPS URLs
-
-In your API server's @application.yml@ file, add the following entry:
-
-<notextile>
-<pre><code>git_repo_https_base: https://git.<span class="userinput">uuid_prefix.your.domain</span>/
-</code></pre>
-</notextile>
-
-Make sure to include the trailing slash.
-
h2. Restart Nginx
Restart Nginx to make the Nginx and API server configuration changes take effect.
<notextile>
<pre><code>~$ <span class="userinput">keepproxy -h</span>
-...
-Usage: keepproxy [-config path/to/keepproxy.yml]
-...
+Usage of keepproxy:
+ -config file
+ Site configuration file (default may be overridden by setting an ARVADOS_CONFIG environment variable) (default "/etc/arvados/config.yml")
+ -dump-config
+ write current configuration to stdout and exit
+[...]
+ -version
+ print version information and exit.
</code></pre>
</notextile>
-h3. Create an API token for the Keepproxy server
-
-{% assign railscmd = "bundle exec ./script/get_anonymous_user_token.rb --get" %}
-{% assign railsout = "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" %}
-The Keepproxy server needs a token to talk to the API server. On the <strong>API server</strong>, use the following command to create the token. {% include 'install_rails_command' %}
-
-h3. Set up the Keepproxy service
+h3. Update the cluster config
-Install runit to supervise the keepproxy daemon. {% include 'install_runit' %}
-
-The run script for the keepproxy service should set the environment variables @ARVADOS_API_TOKEN@ (with the token you just generated), @ARVADOS_API_HOST@, and, if needed, @ARVADOS_API_HOST_INSECURE@. The core keepproxy command to run is:
+Edit the cluster config at @/etc/arvados/config.yml@ and set @Services.Keepproxy.ExternalURL@ and @Services.Keepproxy.InternalURLs@. Replace @uuid_prefix@ with your cluster id.
<notextile>
-<pre><code>ARVADOS_API_TOKEN=<span class="userinput">{{railsout}}</span> ARVADOS_API_HOST=<span class="userinput">uuid_prefix.your.domain</span> exec keepproxy
-</code></pre>
+<pre><code>Clusters:
+ <span class="userinput">uuid_prefix</span>:
+ Services:
+ Keepproxy:
+ ExternalURL: <span class="userinput">https://keep.uuid_prefix.your.domain</span>
+ InternalURLs:
+ <span class="userinput">"http://localhost:25107": {}</span>
+</span></code></pre>
</notextile>
h3. Set up a reverse proxy with SSL support
EOF</span>
</code></pre></notextile>
+h2. Run Keepproxy
+
+h3. Start the service (option 1: systemd)
+
+If your system does not use systemd, skip this section and follow the "runit instructions":#runit instead.
+
+If your system uses systemd, the keepproxy service should already be set up. Start it and check its status:
+
+<notextile>
+<pre><code>~$ <span class="userinput">sudo systemctl restart keepproxy</span>
+~$ <span class="userinput">sudo systemctl status keepproxy</span>
+● keepproxy.service - Arvados Keep Proxy
+ Loaded: loaded (/lib/systemd/system/keepproxy.service; enabled)
+ Active: active (running) since Tue 2019-07-23 09:33:47 EDT; 3 weeks 1 days ago
+ Docs: https://doc.arvados.org/
+ Main PID: 1150 (Keepproxy)
+ CGroup: /system.slice/keepproxy.service
+ └─1150 /usr/bin/keepproxy
+[...]
+</code></pre>
+</notextile>
+
+h3(#runit). Start the service (option 2: runit)
+
+Install runit to supervise the Keep-web daemon. {% include 'install_runit' %}
+
h3. Testing keepproxy
Log into a host that is on an external network from your private Arvados network. The host should be able to contact your keepproxy server (eg keep.$uuid_prefix.arvadosapi.com), but not your keepstore servers (eg keep[0-9].$uuid_prefix.arvadosapi.com).
WebsocketClientEventQueue: 64
WebsocketServerEventQueue: 4
+ # Timeout on requests to internal Keep services.
+ KeepServiceRequestTimeout: 15s
+
Users:
# Config parameters to automatically setup new users. If enabled,
# this users will be able to self-activate. Enable this if you want
RemoteTokenRefresh: 5m
Git:
+ # Path to git or gitolite-shell executable. Each authenticated
+ # request will execute this program with the single argument "http-backend"
+ GitCommand: /usr/bin/git
+
+ # Path to Gitolite's home directory. If a non-empty path is given,
+ # the CGI environment will be set up to support the use of
+ # gitolite-shell as a GitCommand: for example, if GitoliteHome is
+ # "/gh", then the CGI environment will have GITOLITE_HTTP_HOME=/gh,
+ # PATH=$PATH:/gh/bin, and GL_BYPASS_ACCESS_CHECKS=1.
+ GitoliteHome: ""
+
# Git repositories must be readable by api server, or you won't be
# able to submit crunch jobs. To pass the test suites, put a clone
# of the arvados tree in {git_repositories_dir}/arvados.git or
return nil
}
+type oldKeepProxyConfig struct {
+ Client *arvados.Client
+ Listen *string
+ DisableGet *bool
+ DisablePut *bool
+ DefaultReplicas *int
+ Timeout *arvados.Duration
+ PIDFile *string
+ Debug *bool
+ ManagementToken *string
+}
+
+const defaultKeepproxyConfigPath = "/etc/arvados/keepproxy/keepproxy.yml"
+
+func (ldr *Loader) loadOldKeepproxyConfig(cfg *arvados.Config) error {
+ if ldr.KeepproxyPath == "" {
+ return nil
+ }
+ var oc oldKeepProxyConfig
+ err := ldr.loadOldConfigHelper("keepproxy", ldr.KeepproxyPath, &oc)
+ if os.IsNotExist(err) && ldr.KeepproxyPath == defaultKeepproxyConfigPath {
+ return nil
+ } else if err != nil {
+ return err
+ }
+
+ cluster, err := cfg.GetCluster("")
+ if err != nil {
+ return err
+ }
+
+ loadOldClientConfig(cluster, oc.Client)
+
+ if oc.Listen != nil {
+ cluster.Services.Keepproxy.InternalURLs[arvados.URL{Host: *oc.Listen}] = arvados.ServiceInstance{}
+ }
+ if oc.DefaultReplicas != nil {
+ cluster.Collections.DefaultReplication = *oc.DefaultReplicas
+ }
+ if oc.Timeout != nil {
+ cluster.API.KeepServiceRequestTimeout = *oc.Timeout
+ }
+ if oc.Debug != nil {
+ if *oc.Debug && cluster.SystemLogs.LogLevel != "debug" {
+ cluster.SystemLogs.LogLevel = "debug"
+ } else if !*oc.Debug && cluster.SystemLogs.LogLevel != "info" {
+ cluster.SystemLogs.LogLevel = "info"
+ }
+ }
+ if oc.ManagementToken != nil {
+ cluster.ManagementToken = *oc.ManagementToken
+ }
+
+ // The following legacy options are no longer supported. If they are set to
+ // true or PIDFile has a value, error out and notify the user
+ unsupportedEntry := func(cfgEntry string) error {
+ return fmt.Errorf("the keepproxy %s configuration option is no longer supported, please remove it from your configuration file", cfgEntry)
+ }
+ if oc.DisableGet != nil && *oc.DisableGet {
+ return unsupportedEntry("DisableGet")
+ }
+ if oc.DisablePut != nil && *oc.DisablePut {
+ return unsupportedEntry("DisablePut")
+ }
+ if oc.PIDFile != nil && *oc.PIDFile != "" {
+ return unsupportedEntry("PIDFile")
+ }
+
+ cfg.Clusters[cluster.ClusterID] = *cluster
+ return nil
+}
+
const defaultKeepWebConfigPath = "/etc/arvados/keep-web/keep-web.yml"
type oldKeepWebConfig struct {
cfg.Clusters[cluster.ClusterID] = *cluster
return nil
}
+
+const defaultGitHttpdConfigPath = "/etc/arvados/git-httpd/git-httpd.yml"
+
+type oldGitHttpdConfig struct {
+ Client *arvados.Client
+ Listen string
+ GitCommand string
+ GitoliteHome string
+ RepoRoot string
+ ManagementToken string
+}
+
+func (ldr *Loader) loadOldGitHttpdConfig(cfg *arvados.Config) error {
+ if ldr.GitHttpdPath == "" {
+ return nil
+ }
+ var oc oldGitHttpdConfig
+ err := ldr.loadOldConfigHelper("arv-git-httpd", ldr.GitHttpdPath, &oc)
+ if os.IsNotExist(err) && ldr.GitHttpdPath == defaultGitHttpdConfigPath {
+ return nil
+ } else if err != nil {
+ return err
+ }
+
+ cluster, err := cfg.GetCluster("")
+ if err != nil {
+ return err
+ }
+
+ loadOldClientConfig(cluster, oc.Client)
+
+ cluster.Services.GitHTTP.InternalURLs[arvados.URL{Host: oc.Listen}] = arvados.ServiceInstance{}
+ cluster.TLS.Insecure = oc.Client.Insecure
+ cluster.ManagementToken = oc.ManagementToken
+ cluster.Git.GitCommand = oc.GitCommand
+ cluster.Git.GitoliteHome = oc.GitoliteHome
+ cluster.Git.Repositories = oc.RepoRoot
+
+ cfg.Clusters[cluster.ClusterID] = *cluster
+ return nil
+}
import (
"flag"
+ "fmt"
"io/ioutil"
"os"
"time"
check "gopkg.in/check.v1"
)
+func testLoadLegacyConfig(content []byte, mungeFlag string, c *check.C) (*arvados.Cluster, error) {
+ tmpfile, err := ioutil.TempFile("", "example")
+ if err != nil {
+ return nil, err
+ }
+ defer os.Remove(tmpfile.Name())
+
+ if _, err := tmpfile.Write(content); err != nil {
+ return nil, err
+ }
+ if err := tmpfile.Close(); err != nil {
+ return nil, err
+ }
+ flags := flag.NewFlagSet("test", flag.ExitOnError)
+ ldr := testLoader(c, "Clusters: {zzzzz: {}}", nil)
+ ldr.SetupFlags(flags)
+ args := ldr.MungeLegacyConfigArgs(ldr.Logger, []string{"-config", tmpfile.Name()}, mungeFlag)
+ flags.Parse(args)
+ cfg, err := ldr.Load()
+ if err != nil {
+ return nil, err
+ }
+ cluster, err := cfg.GetCluster("")
+ if err != nil {
+ return nil, err
+ }
+ return cluster, nil
+}
+
func (s *LoadSuite) TestDeprecatedNodeProfilesToServices(c *check.C) {
hostname, err := os.Hostname()
c.Assert(err, check.IsNil)
"ManagementToken": "xyzzy"
}
`)
- tmpfile, err := ioutil.TempFile("", "example")
- if err != nil {
- c.Error(err)
- }
- defer os.Remove(tmpfile.Name())
-
- if _, err := tmpfile.Write(content); err != nil {
- c.Error(err)
- }
- if err := tmpfile.Close(); err != nil {
- c.Error(err)
- }
- flags := flag.NewFlagSet("keep-web", flag.ExitOnError)
- ldr := testLoader(c, "Clusters: {zzzzz: {}}", nil)
- ldr.SetupFlags(flags)
- args := ldr.MungeLegacyConfigArgs(ldr.Logger, []string{"-config", tmpfile.Name()}, "-legacy-keepweb-config")
- flags.Parse(args)
- cfg, err := ldr.Load()
- if err != nil {
- c.Error(err)
- }
- c.Check(cfg, check.NotNil)
- cluster, err := cfg.GetCluster("")
- if err != nil {
- c.Error(err)
- }
- c.Check(cluster, check.NotNil)
+ cluster, err := testLoadLegacyConfig(content, "-legacy-keepweb-config", c)
+ c.Check(err, check.IsNil)
c.Check(cluster.Services.Controller.ExternalURL, check.Equals, arvados.URL{Scheme: "https", Host: "example.com"})
c.Check(cluster.SystemRootToken, check.Equals, "abcdefg")
c.Check(cluster.Users.AnonymousUserToken, check.Equals, "anonusertoken")
c.Check(cluster.ManagementToken, check.Equals, "xyzzy")
}
+
+func (s *LoadSuite) TestLegacyKeepproxyConfig(c *check.C) {
+ f := "-legacy-keepproxy-config"
+ content := []byte(fmtKeepproxyConfig("", true))
+ cluster, err := testLoadLegacyConfig(content, f, c)
+
+ c.Check(err, check.IsNil)
+ c.Check(cluster, check.NotNil)
+ c.Check(cluster.Services.Controller.ExternalURL, check.Equals, arvados.URL{Scheme: "https", Host: "example.com"})
+ c.Check(cluster.SystemRootToken, check.Equals, "abcdefg")
+ c.Check(cluster.ManagementToken, check.Equals, "xyzzy")
+ c.Check(cluster.Services.Keepproxy.InternalURLs[arvados.URL{Host: ":80"}], check.Equals, arvados.ServiceInstance{})
+ c.Check(cluster.Collections.DefaultReplication, check.Equals, 0)
+ c.Check(cluster.API.KeepServiceRequestTimeout.String(), check.Equals, "15s")
+ c.Check(cluster.SystemLogs.LogLevel, check.Equals, "debug")
+
+ content = []byte(fmtKeepproxyConfig("", false))
+ cluster, err = testLoadLegacyConfig(content, f, c)
+ c.Check(cluster.SystemLogs.LogLevel, check.Equals, "info")
+
+ content = []byte(fmtKeepproxyConfig(`"DisableGet": true,`, true))
+ _, err = testLoadLegacyConfig(content, f, c)
+ c.Check(err, check.NotNil)
+
+ content = []byte(fmtKeepproxyConfig(`"DisablePut": true,`, true))
+ _, err = testLoadLegacyConfig(content, f, c)
+ c.Check(err, check.NotNil)
+
+ content = []byte(fmtKeepproxyConfig(`"PIDFile": "test",`, true))
+ _, err = testLoadLegacyConfig(content, f, c)
+ c.Check(err, check.NotNil)
+
+ content = []byte(fmtKeepproxyConfig(`"DisableGet": false, "DisablePut": false, "PIDFile": "",`, true))
+ _, err = testLoadLegacyConfig(content, f, c)
+ c.Check(err, check.IsNil)
+}
+
+func fmtKeepproxyConfig(param string, debugLog bool) string {
+ return fmt.Sprintf(`
+{
+ "Client": {
+ "Scheme": "",
+ "APIHost": "example.com",
+ "AuthToken": "abcdefg",
+ "Insecure": false
+ },
+ "Listen": ":80",
+ "DefaultReplicas": 0,
+ "Timeout": "15s",
+ "Debug": %t,
+ %s
+ "ManagementToken": "xyzzy"
+}
+`, debugLog, param)
+}
+
+func (s *LoadSuite) TestLegacyArvGitHttpdConfig(c *check.C) {
+ content := []byte(`
+{
+ "Client": {
+ "Scheme": "",
+ "APIHost": "example.com",
+ "AuthToken": "abcdefg",
+ },
+ "Listen": ":9000",
+ "GitCommand": "/test/git",
+ "GitoliteHome": "/test/gitolite",
+ "RepoRoot": "/test/reporoot",
+ "ManagementToken": "xyzzy"
+}
+`)
+ f := "-legacy-git-httpd-config"
+ cluster, err := testLoadLegacyConfig(content, f, c)
+
+ c.Check(err, check.IsNil)
+ c.Check(cluster, check.NotNil)
+ c.Check(cluster.Services.Controller.ExternalURL, check.Equals, arvados.URL{Scheme: "https", Host: "example.com"})
+ c.Check(cluster.SystemRootToken, check.Equals, "abcdefg")
+ c.Check(cluster.ManagementToken, check.Equals, "xyzzy")
+ c.Check(cluster.Git.GitCommand, check.Equals, "/test/git")
+ c.Check(cluster.Git.GitoliteHome, check.Equals, "/test/gitolite")
+ c.Check(cluster.Git.Repositories, check.Equals, "/test/reporoot")
+ c.Check(cluster.Services.Keepproxy.InternalURLs[arvados.URL{Host: ":9000"}], check.Equals, arvados.ServiceInstance{})
+}
"API.WebsocketClientEventQueue": false,
"API.SendTimeout": true,
"API.WebsocketServerEventQueue": false,
+ "API.KeepServiceRequestTimeout": false,
"AuditLogs": false,
"AuditLogs.MaxAge": false,
"AuditLogs.MaxDeleteBatch": false,
WebsocketClientEventQueue: 64
WebsocketServerEventQueue: 4
+ # Timeout on requests to internal Keep services.
+ KeepServiceRequestTimeout: 15s
+
Users:
# Config parameters to automatically setup new users. If enabled,
# this users will be able to self-activate. Enable this if you want
RemoteTokenRefresh: 5m
Git:
+ # Path to git or gitolite-shell executable. Each authenticated
+ # request will execute this program with the single argument "http-backend"
+ GitCommand: /usr/bin/git
+
+ # Path to Gitolite's home directory. If a non-empty path is given,
+ # the CGI environment will be set up to support the use of
+ # gitolite-shell as a GitCommand: for example, if GitoliteHome is
+ # "/gh", then the CGI environment will have GITOLITE_HTTP_HOME=/gh,
+ # PATH=$PATH:/gh/bin, and GL_BYPASS_ACCESS_CHECKS=1.
+ GitoliteHome: ""
+
# Git repositories must be readable by api server, or you won't be
# able to submit crunch jobs. To pass the test suites, put a clone
# of the arvados tree in {git_repositories_dir}/arvados.git or
KeepWebPath string
CrunchDispatchSlurmPath string
WebsocketPath string
+ KeepproxyPath string
+ GitHttpdPath string
configdata []byte
}
flagset.StringVar(&ldr.KeepWebPath, "legacy-keepweb-config", defaultKeepWebConfigPath, "Legacy keep-web configuration `file`")
flagset.StringVar(&ldr.CrunchDispatchSlurmPath, "legacy-crunch-dispatch-slurm-config", defaultCrunchDispatchSlurmConfigPath, "Legacy crunch-dispatch-slurm configuration `file`")
flagset.StringVar(&ldr.WebsocketPath, "legacy-ws-config", defaultWebsocketConfigPath, "Legacy arvados-ws configuration `file`")
+ flagset.StringVar(&ldr.KeepproxyPath, "legacy-keepproxy-config", defaultKeepproxyConfigPath, "Legacy keepproxy configuration `file`")
+ flagset.StringVar(&ldr.GitHttpdPath, "legacy-git-httpd-config", defaultGitHttpdConfigPath, "Legacy arv-git-httpd configuration `file`")
flagset.BoolVar(&ldr.SkipLegacy, "skip-legacy", false, "Don't load legacy config files")
}
if legacyConfigArg != "-legacy-keepweb-config" {
ldr.KeepWebPath = ""
}
+ if legacyConfigArg != "-legacy-keepproxy-config" {
+ ldr.KeepproxyPath = ""
+ }
+ if legacyConfigArg != "-legacy-git-httpd-config" {
+ ldr.GitHttpdPath = ""
+ }
return munged
}
ldr.loadOldKeepWebConfig(&cfg),
ldr.loadOldCrunchDispatchSlurmConfig(&cfg),
ldr.loadOldWebsocketConfig(&cfg),
+ ldr.loadOldKeepproxyConfig(&cfg),
+ ldr.loadOldGitHttpdConfig(&cfg),
} {
if err != nil {
return nil, err
SendTimeout Duration
WebsocketClientEventQueue int
WebsocketServerEventQueue int
+ KeepServiceRequestTimeout Duration
}
AuditLogs struct {
MaxAge Duration
WebDAVCache WebDAVCacheConfig
}
Git struct {
+ GitCommand string
+ GitoliteHome string
Repositories string
}
Login struct {
from arvados.collection import CollectionReader
HEX_RE = re.compile(r'^[0-9a-fA-F]+$')
+CR_UNCOMMITTED = 'Uncommitted'
+CR_COMMITTED = 'Committed'
+CR_FINAL = 'Final'
keep_locator_pattern = re.compile(r'[0-9a-f]{32}\+\d+(\+\S+)*')
signed_locator_pattern = re.compile(r'[0-9a-f]{32}\+\d+(\+\S+)*\+A\S+(\+\S+)*')
env['ARVADOS_API_TOKEN'] = auth_token('anonymous')
logf = open(_logfilename('keepproxy'), 'a')
kp = subprocess.Popen(
- ['keepproxy',
- '-pid='+_pidfile('keepproxy'),
- '-listen=:{}'.format(port)],
- env=env, stdin=open('/dev/null'), stdout=logf, stderr=logf, close_fds=True)
+ ['keepproxy'], env=env, stdin=open('/dev/null'), stdout=logf, stderr=logf, close_fds=True)
+
+ with open(_pidfile('keepproxy'), 'w') as f:
+ f.write(str(kp.pid))
+ _wait_until_port_listens(port)
print("Using API %s token %s" % (os.environ['ARVADOS_API_HOST'], auth_token('admin')), file=sys.stdout)
api = arvados.api(
return
stop_arv_git_httpd()
- gitdir = os.path.join(SERVICES_SRC_DIR, 'api', 'tmp', 'git')
gitport = internal_port_from_config("GitHTTP")
env = os.environ.copy()
env.pop('ARVADOS_API_TOKEN', None)
logf = open(_logfilename('arv-git-httpd'), 'a')
- agh = subprocess.Popen(
- ['arv-git-httpd',
- '-repo-root='+gitdir+'/test',
- '-management-token=e687950a23c3a9bceec28c6223a06c79',
- '-address=:'+str(gitport)],
+ agh = subprocess.Popen(['arv-git-httpd'],
env=env, stdin=open('/dev/null'), stdout=logf, stderr=logf)
with open(_pidfile('arv-git-httpd'), 'w') as f:
f.write(str(agh.pid))
},
"Collections": {
"TrustAllContent": True
+ },
+ "Git": {
+ "Repositories": "%s/test" % os.path.join(SERVICES_SRC_DIR, 'api', 'tmp', 'git')
}
}
}
Description=Arvados git server
Documentation=https://doc.arvados.org/
After=network.target
-AssertPathExists=/etc/arvados/git-httpd/git-httpd.yml
# systemd==229 (ubuntu:xenial) obeys StartLimitInterval in the [Unit] section
StartLimitInterval=0
"sync"
"time"
+ "git.curoverse.com/arvados.git/sdk/go/arvados"
"git.curoverse.com/arvados.git/sdk/go/arvadosclient"
"git.curoverse.com/arvados.git/sdk/go/auth"
"git.curoverse.com/arvados.git/sdk/go/httpserver"
type authHandler struct {
handler http.Handler
clientPool *arvadosclient.ClientPool
+ cluster *arvados.Cluster
setupOnce sync.Once
}
func (h *authHandler) setup() {
- ac, err := arvadosclient.New(&theConfig.Client)
+ client, err := arvados.NewClientFromConfig(h.cluster)
if err != nil {
log.Fatal(err)
}
+
+ ac, err := arvadosclient.New(client)
+ if err != nil {
+ log.Fatalf("Error setting up arvados client prototype %v", err)
+ }
+
h.clientPool = &arvadosclient.ClientPool{Prototype: ac}
}
"/" + repoName + "/.git",
}
for _, dir := range tryDirs {
- if fileInfo, err := os.Stat(theConfig.RepoRoot + dir); err != nil {
+ if fileInfo, err := os.Stat(h.cluster.Git.Repositories + dir); err != nil {
if !os.IsNotExist(err) {
statusCode, statusText = http.StatusInternalServerError, err.Error()
return
}
if rewrittenPath == "" {
log.Println("WARNING:", repoUUID,
- "git directory not found in", theConfig.RepoRoot, tryDirs)
+ "git directory not found in", h.cluster.Git.Repositories, tryDirs)
// We say "content not found" to disambiguate from the
// earlier "API says that repo does not exist" error.
statusCode, statusText = http.StatusNotFound, "content not found"
"path/filepath"
"strings"
+ "git.curoverse.com/arvados.git/lib/config"
"git.curoverse.com/arvados.git/sdk/go/arvados"
"git.curoverse.com/arvados.git/sdk/go/arvadostest"
check "gopkg.in/check.v1"
var _ = check.Suite(&AuthHandlerSuite{})
-type AuthHandlerSuite struct{}
+type AuthHandlerSuite struct {
+ cluster *arvados.Cluster
+}
func (s *AuthHandlerSuite) SetUpSuite(c *check.C) {
arvadostest.StartAPI()
arvadostest.ResetEnv()
repoRoot, err := filepath.Abs("../api/tmp/git/test")
c.Assert(err, check.IsNil)
- theConfig = &Config{
- Client: arvados.Client{
- APIHost: arvadostest.APIHost(),
- Insecure: true,
- },
- Listen: ":0",
- GitCommand: "/usr/bin/git",
- RepoRoot: repoRoot,
- ManagementToken: arvadostest.ManagementToken,
- }
+
+ cfg, err := config.NewLoader(nil, nil).Load()
+ c.Assert(err, check.Equals, nil)
+ s.cluster, err = cfg.GetCluster("")
+ c.Assert(err, check.Equals, nil)
+
+ s.cluster.Services.GitHTTP.InternalURLs = map[arvados.URL]arvados.ServiceInstance{arvados.URL{Host: "localhost:0"}: arvados.ServiceInstance{}}
+ s.cluster.TLS.Insecure = true
+ s.cluster.Git.GitCommand = "/usr/bin/git"
+ s.cluster.Git.Repositories = repoRoot
}
func (s *AuthHandlerSuite) TestPermission(c *check.C) {
h := &authHandler{handler: http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
log.Printf("%v", r.URL)
io.WriteString(w, r.URL.Path)
- })}
+ }), cluster: s.cluster}
baseURL, err := url.Parse("http://git.example/")
c.Assert(err, check.IsNil)
for _, trial := range []struct {
}
func (s *AuthHandlerSuite) TestCORS(c *check.C) {
- h := &authHandler{}
+ h := &authHandler{cluster: s.cluster}
// CORS preflight
resp := httptest.NewRecorder()
"net/http"
"net/http/cgi"
"os"
+
+ "git.curoverse.com/arvados.git/sdk/go/arvados"
)
// gitHandler is an http.Handler that invokes git-http-backend (or
cgi.Handler
}
-func newGitHandler() http.Handler {
+func newGitHandler(cluster *arvados.Cluster) http.Handler {
const glBypass = "GL_BYPASS_ACCESS_CHECKS"
const glHome = "GITOLITE_HTTP_HOME"
var env []string
path := os.Getenv("PATH")
- if theConfig.GitoliteHome != "" {
+ if cluster.Git.GitoliteHome != "" {
env = append(env,
- glHome+"="+theConfig.GitoliteHome,
+ glHome+"="+cluster.Git.GitoliteHome,
glBypass+"=1")
- path = path + ":" + theConfig.GitoliteHome + "/bin"
+ path = path + ":" + cluster.Git.GitoliteHome + "/bin"
} else if home, bypass := os.Getenv(glHome), os.Getenv(glBypass); home != "" || bypass != "" {
env = append(env, glHome+"="+home, glBypass+"="+bypass)
log.Printf("DEPRECATED: Passing through %s and %s environment variables. Use GitoliteHome configuration instead.", glHome, glBypass)
}
+
+ var listen arvados.URL
+ for listen = range cluster.Services.GitHTTP.InternalURLs {
+ break
+ }
env = append(env,
- "GIT_PROJECT_ROOT="+theConfig.RepoRoot,
+ "GIT_PROJECT_ROOT="+cluster.Git.Repositories,
"GIT_HTTP_EXPORT_ALL=",
- "SERVER_ADDR="+theConfig.Listen,
+ "SERVER_ADDR="+listen.Host,
"PATH="+path)
return &gitHandler{
Handler: cgi.Handler{
- Path: theConfig.GitCommand,
- Dir: theConfig.RepoRoot,
+ Path: cluster.Git.GitCommand,
+ Dir: cluster.Git.Repositories,
Env: env,
Args: []string{"http-backend"},
},
"net/url"
"regexp"
+ "git.curoverse.com/arvados.git/lib/config"
+ "git.curoverse.com/arvados.git/sdk/go/arvados"
check "gopkg.in/check.v1"
)
var _ = check.Suite(&GitHandlerSuite{})
-type GitHandlerSuite struct{}
+type GitHandlerSuite struct {
+ cluster *arvados.Cluster
+}
-func (s *GitHandlerSuite) TestEnvVars(c *check.C) {
- theConfig = defaultConfig()
- theConfig.RepoRoot = "/"
- theConfig.GitoliteHome = "/test/ghh"
+func (s *GitHandlerSuite) SetUpTest(c *check.C) {
+ cfg, err := config.NewLoader(nil, nil).Load()
+ c.Assert(err, check.Equals, nil)
+ s.cluster, err = cfg.GetCluster("")
+ c.Assert(err, check.Equals, nil)
+ s.cluster.Services.GitHTTP.InternalURLs = map[arvados.URL]arvados.ServiceInstance{arvados.URL{Host: "localhost:80"}: arvados.ServiceInstance{}}
+ s.cluster.Git.GitoliteHome = "/test/ghh"
+ s.cluster.Git.Repositories = "/"
+}
+
+func (s *GitHandlerSuite) TestEnvVars(c *check.C) {
u, err := url.Parse("git.zzzzz.arvadosapi.com/test")
c.Check(err, check.Equals, nil)
resp := httptest.NewRecorder()
URL: u,
RemoteAddr: "[::1]:12345",
}
- h := newGitHandler()
+ h := newGitHandler(s.cluster)
h.(*gitHandler).Path = "/bin/sh"
h.(*gitHandler).Args = []string{"-c", "printf 'Content-Type: text/plain\r\n\r\n'; env"}
c.Check(body, check.Matches, `(?ms).*^GL_BYPASS_ACCESS_CHECKS=1$.*`)
c.Check(body, check.Matches, `(?ms).*^REMOTE_HOST=::1$.*`)
c.Check(body, check.Matches, `(?ms).*^REMOTE_PORT=12345$.*`)
- c.Check(body, check.Matches, `(?ms).*^SERVER_ADDR=`+regexp.QuoteMeta(theConfig.Listen)+`$.*`)
+ c.Check(body, check.Matches, `(?ms).*^SERVER_ADDR=`+regexp.QuoteMeta("localhost:80")+`$.*`)
}
func (s *GitHandlerSuite) TestCGIErrorOnSplitHostPortError(c *check.C) {
URL: u,
RemoteAddr: "test.bad.address.missing.port",
}
- h := newGitHandler()
+ h := newGitHandler(s.cluster)
h.ServeHTTP(resp, req)
c.Check(resp.Code, check.Equals, http.StatusInternalServerError)
c.Check(resp.Body.String(), check.Equals, "")
"os/exec"
"strings"
+ "git.curoverse.com/arvados.git/lib/config"
"git.curoverse.com/arvados.git/sdk/go/arvados"
- "git.curoverse.com/arvados.git/sdk/go/arvadostest"
check "gopkg.in/check.v1"
)
runGitolite("gitolite", "setup", "--admin", "root")
s.tmpRepoRoot = s.gitoliteHome + "/repositories"
- s.Config = &Config{
- Client: arvados.Client{
- APIHost: arvadostest.APIHost(),
- Insecure: true,
- },
- Listen: "localhost:0",
- GitCommand: "/usr/share/gitolite3/gitolite-shell",
- GitoliteHome: s.gitoliteHome,
- RepoRoot: s.tmpRepoRoot,
- }
+
+ cfg, err := config.NewLoader(nil, nil).Load()
+ c.Assert(err, check.Equals, nil)
+ s.cluster, err = cfg.GetCluster("")
+ c.Assert(err, check.Equals, nil)
+
+ s.cluster.Services.GitHTTP.InternalURLs = map[arvados.URL]arvados.ServiceInstance{arvados.URL{Host: "localhost:0"}: arvados.ServiceInstance{}}
+ s.cluster.TLS.Insecure = true
+ s.cluster.Git.GitCommand = "/usr/share/gitolite3/gitolite-shell"
+ s.cluster.Git.GitoliteHome = s.gitoliteHome
+ s.cluster.Git.Repositories = s.tmpRepoRoot
+
s.IntegrationSuite.SetUpTest(c)
// Install the gitolite hooks in the bare repo we made in
"strings"
"testing"
+ "git.curoverse.com/arvados.git/lib/config"
"git.curoverse.com/arvados.git/sdk/go/arvados"
"git.curoverse.com/arvados.git/sdk/go/arvadostest"
check "gopkg.in/check.v1"
tmpRepoRoot string
tmpWorkdir string
testServer *server
- Config *Config
+ cluster *arvados.Cluster
}
func (s *IntegrationSuite) SetUpSuite(c *check.C) {
func (s *IntegrationSuite) SetUpTest(c *check.C) {
arvadostest.ResetEnv()
- s.testServer = &server{}
+
var err error
if s.tmpRepoRoot == "" {
s.tmpRepoRoot, err = ioutil.TempDir("", "arv-git-httpd")
_, err = exec.Command("sh", "-c", "cd "+s.tmpWorkdir+" && echo work >work && git add work && git -c user.name=Foo -c user.email=Foo commit -am 'workdir: test'").CombinedOutput()
c.Assert(err, check.Equals, nil)
+ if s.cluster == nil {
+ cfg, err := config.NewLoader(nil, nil).Load()
+ c.Assert(err, check.Equals, nil)
+ s.cluster, err = cfg.GetCluster("")
+ c.Assert(err, check.Equals, nil)
+
+ s.cluster.Services.GitHTTP.InternalURLs = map[arvados.URL]arvados.ServiceInstance{arvados.URL{Host: "localhost:0"}: arvados.ServiceInstance{}}
+ s.cluster.TLS.Insecure = true
+ s.cluster.Git.GitCommand = "/usr/bin/git"
+ s.cluster.Git.Repositories = s.tmpRepoRoot
+ s.cluster.ManagementToken = arvadostest.ManagementToken
+ }
+
+ s.testServer = &server{cluster: s.cluster}
+ err = s.testServer.Start()
+ c.Assert(err, check.Equals, nil)
+
_, err = exec.Command("git", "config",
"--file", s.tmpWorkdir+"/.git/config",
"credential.http://"+s.testServer.Addr+"/.helper",
"none").Output()
c.Assert(err, check.Equals, nil)
- if s.Config == nil {
- s.Config = &Config{
- Client: arvados.Client{
- APIHost: arvadostest.APIHost(),
- Insecure: true,
- },
- Listen: "localhost:0",
- GitCommand: "/usr/bin/git",
- RepoRoot: s.tmpRepoRoot,
- ManagementToken: arvadostest.ManagementToken,
- }
- }
-
// Clear ARVADOS_API_* env vars before starting up the server,
// to make sure arv-git-httpd doesn't use them or complain
// about them being missing.
os.Unsetenv("ARVADOS_API_HOST")
os.Unsetenv("ARVADOS_API_HOST_INSECURE")
os.Unsetenv("ARVADOS_API_TOKEN")
-
- theConfig = s.Config
- err = s.testServer.Start()
- c.Assert(err, check.Equals, nil)
}
func (s *IntegrationSuite) TearDownTest(c *check.C) {
}
s.tmpWorkdir = ""
- s.Config = nil
-
- theConfig = defaultConfig()
+ s.cluster = nil
}
func (s *IntegrationSuite) RunGit(c *check.C, token, gitCmd, repo string, args ...string) error {
package main
import (
- "encoding/json"
"flag"
"fmt"
- "log"
"os"
- "regexp"
- "git.curoverse.com/arvados.git/sdk/go/arvados"
- "git.curoverse.com/arvados.git/sdk/go/config"
+ "git.curoverse.com/arvados.git/lib/config"
"github.com/coreos/go-systemd/daemon"
+ "github.com/ghodss/yaml"
+ log "github.com/sirupsen/logrus"
)
var version = "dev"
-// Server configuration
-type Config struct {
- Client arvados.Client
- Listen string
- GitCommand string
- RepoRoot string
- GitoliteHome string
- ManagementToken string
-}
-
-var theConfig = defaultConfig()
-
-func defaultConfig() *Config {
- return &Config{
- Listen: ":80",
- GitCommand: "/usr/bin/git",
- RepoRoot: "/var/lib/arvados/git/repositories",
- }
-}
-
func main() {
- const defaultCfgPath = "/etc/arvados/git-httpd/git-httpd.yml"
- const deprecated = " (DEPRECATED -- use config file instead)"
- flag.StringVar(&theConfig.Listen, "address", theConfig.Listen,
- "Address to listen on, \"host:port\" or \":port\"."+deprecated)
- flag.StringVar(&theConfig.GitCommand, "git-command", theConfig.GitCommand,
- "Path to git or gitolite-shell executable. Each authenticated request will execute this program with a single argument, \"http-backend\"."+deprecated)
- flag.StringVar(&theConfig.RepoRoot, "repo-root", theConfig.RepoRoot,
- "Path to git repositories."+deprecated)
- flag.StringVar(&theConfig.GitoliteHome, "gitolite-home", theConfig.GitoliteHome,
- "Value for GITOLITE_HTTP_HOME environment variable. If not empty, GL_BYPASS_ACCESS_CHECKS=1 will also be set."+deprecated)
+ logger := log.New()
+ log.SetFormatter(&log.JSONFormatter{
+ TimestampFormat: "2006-01-02T15:04:05.000000000Z07:00",
+ })
- cfgPath := flag.String("config", defaultCfgPath, "Configuration file `path`.")
- dumpConfig := flag.Bool("dump-config", false, "write current configuration to stdout and exit (useful for migrating from command line flags to config file)")
- getVersion := flag.Bool("version", false, "print version information and exit.")
+ flags := flag.NewFlagSet(os.Args[0], flag.ExitOnError)
+ loader := config.NewLoader(os.Stdin, logger)
+ loader.SetupFlags(flags)
- flag.StringVar(&theConfig.ManagementToken, "management-token", theConfig.ManagementToken,
- "Authorization token to be included in all health check requests.")
+ dumpConfig := flags.Bool("dump-config", false, "write current configuration to stdout and exit (useful for migrating from command line flags to config file)")
+ getVersion := flags.Bool("version", false, "print version information and exit.")
- flag.Usage = usage
- flag.Parse()
+ args := loader.MungeLegacyConfigArgs(logger, os.Args[1:], "-legacy-git-httpd-config")
+ flags.Parse(args)
- // Print version information if requested
if *getVersion {
fmt.Printf("arv-git-httpd %s\n", version)
return
}
- err := config.LoadFile(theConfig, *cfgPath)
+ cfg, err := loader.Load()
if err != nil {
- h := os.Getenv("ARVADOS_API_HOST")
- if h == "" || !os.IsNotExist(err) || *cfgPath != defaultCfgPath {
- log.Fatal(err)
- }
- log.Print("DEPRECATED: No config file found, but ARVADOS_API_HOST environment variable is set. Please use a config file instead.")
- theConfig.Client.APIHost = h
- if regexp.MustCompile("^(?i:1|yes|true)$").MatchString(os.Getenv("ARVADOS_API_HOST_INSECURE")) {
- theConfig.Client.Insecure = true
- }
- if j, err := json.MarshalIndent(theConfig, "", " "); err == nil {
- log.Print("Current configuration:\n", string(j))
- }
+ log.Fatal(err)
+ }
+
+ cluster, err := cfg.GetCluster("")
+ if err != nil {
+ log.Fatal(err)
}
if *dumpConfig {
- log.Fatal(config.DumpAndExit(theConfig))
+ out, err := yaml.Marshal(cfg)
+ if err != nil {
+ log.Fatal(err)
+ }
+ _, err = os.Stdout.Write(out)
+ if err != nil {
+ log.Fatal(err)
+ }
+ return
}
- srv := &server{}
+ srv := &server{cluster: cluster}
if err := srv.Start(); err != nil {
log.Fatal(err)
}
}
log.Printf("arv-git-httpd %s started", version)
log.Println("Listening at", srv.Addr)
- log.Println("Repository root", theConfig.RepoRoot)
+ log.Println("Repository root", cluster.Git.Repositories)
if err := srv.Wait(); err != nil {
log.Fatal(err)
}
import (
"net/http"
+ "git.curoverse.com/arvados.git/sdk/go/arvados"
"git.curoverse.com/arvados.git/sdk/go/health"
"git.curoverse.com/arvados.git/sdk/go/httpserver"
)
type server struct {
httpserver.Server
+ cluster *arvados.Cluster
}
func (srv *server) Start() error {
mux := http.NewServeMux()
- mux.Handle("/", &authHandler{handler: newGitHandler()})
+ mux.Handle("/", &authHandler{handler: newGitHandler(srv.cluster), cluster: srv.cluster})
mux.Handle("/_health/", &health.Handler{
- Token: theConfig.ManagementToken,
+ Token: srv.cluster.ManagementToken,
Prefix: "/_health/",
})
+
+ var listen arvados.URL
+ for listen = range srv.cluster.Services.GitHTTP.InternalURLs {
+ break
+ }
+
srv.Handler = mux
- srv.Addr = theConfig.Listen
+ srv.Addr = listen.Host
return srv.Server.Start()
}
+++ /dev/null
-// Copyright (C) The Arvados Authors. All rights reserved.
-//
-// SPDX-License-Identifier: AGPL-3.0
-
-// arvados-git-httpd provides authenticated access to Arvados-hosted
-// git repositories.
-//
-// See http://doc.arvados.org/install/install-arv-git-httpd.html.
-package main
-
-import (
- "flag"
- "fmt"
- "os"
-
- "github.com/ghodss/yaml"
-)
-
-func usage() {
- c := defaultConfig()
- c.Client.APIHost = "zzzzz.arvadosapi.com:443"
- exampleConfigFile, err := yaml.Marshal(c)
- if err != nil {
- panic(err)
- }
- fmt.Fprintf(os.Stderr, `
-
-arvados-git-httpd provides authenticated access to Arvados-hosted git
-repositories.
-
-See http://doc.arvados.org/install/install-arv-git-httpd.html.
-
-Usage: arvados-git-httpd [-config path/to/arvados/git-httpd.yml]
-
-Options:
-`)
- flag.PrintDefaults()
- fmt.Fprintf(os.Stderr, `
-Example config file:
-
-%s
-
-Client.APIHost:
-
- Address (or address:port) of the Arvados API endpoint.
-
-Client.AuthToken:
-
- Unused. Normally empty, or omitted entirely.
-
-Client.Insecure:
-
- True if your Arvados API endpoint uses an unverifiable SSL/TLS
- certificate.
-
-GitCommand:
-
- Path to git or gitolite-shell executable. Each authenticated
- request will execute this program with the single argument
- "http-backend".
-
-GitoliteHome:
-
- Path to Gitolite's home directory. If a non-empty path is given,
- the CGI environment will be set up to support the use of
- gitolite-shell as a GitCommand: for example, if GitoliteHome is
- "/gh", then the CGI environment will have GITOLITE_HTTP_HOME=/gh,
- PATH=$PATH:/gh/bin, and GL_BYPASS_ACCESS_CHECKS=1.
-
-Listen:
-
- Local port to listen on. Can be "address:port" or ":port", where
- "address" is a host IP address or name and "port" is a port number
- or name.
-
-RepoRoot:
-
- Path to git repositories.
-
-`, exampleConfigFile)
-}
"syscall"
"time"
+ "git.curoverse.com/arvados.git/lib/config"
"git.curoverse.com/arvados.git/sdk/go/arvados"
"git.curoverse.com/arvados.git/sdk/go/arvadosclient"
- "git.curoverse.com/arvados.git/sdk/go/config"
"git.curoverse.com/arvados.git/sdk/go/health"
"git.curoverse.com/arvados.git/sdk/go/httpserver"
"git.curoverse.com/arvados.git/sdk/go/keepclient"
var version = "dev"
-type Config struct {
- Client arvados.Client
- Listen string
- DisableGet bool
- DisablePut bool
- DefaultReplicas int
- Timeout arvados.Duration
- PIDFile string
- Debug bool
- ManagementToken string
-}
-
-func DefaultConfig() *Config {
- return &Config{
- Listen: ":25107",
- Timeout: arvados.Duration(15 * time.Second),
- }
-}
-
var (
listener net.Listener
router http.Handler
const rfc3339NanoFixed = "2006-01-02T15:04:05.000000000Z07:00"
-func main() {
- log.SetFormatter(&log.JSONFormatter{
- TimestampFormat: rfc3339NanoFixed,
- })
+func configure(logger log.FieldLogger, args []string) (*arvados.Cluster, error) {
+ flags := flag.NewFlagSet(args[0], flag.ExitOnError)
- cfg := DefaultConfig()
+ dumpConfig := flags.Bool("dump-config", false, "write current configuration to stdout and exit")
+ getVersion := flags.Bool("version", false, "Print version information and exit.")
- flagset := flag.NewFlagSet("keepproxy", flag.ExitOnError)
- flagset.Usage = usage
+ loader := config.NewLoader(os.Stdin, logger)
+ loader.SetupFlags(flags)
- const deprecated = " (DEPRECATED -- use config file instead)"
- flagset.StringVar(&cfg.Listen, "listen", cfg.Listen, "Local port to listen on."+deprecated)
- flagset.BoolVar(&cfg.DisableGet, "no-get", cfg.DisableGet, "Disable GET operations."+deprecated)
- flagset.BoolVar(&cfg.DisablePut, "no-put", cfg.DisablePut, "Disable PUT operations."+deprecated)
- flagset.IntVar(&cfg.DefaultReplicas, "default-replicas", cfg.DefaultReplicas, "Default number of replicas to write if not specified by the client. If 0, use site default."+deprecated)
- flagset.StringVar(&cfg.PIDFile, "pid", cfg.PIDFile, "Path to write pid file."+deprecated)
- timeoutSeconds := flagset.Int("timeout", int(time.Duration(cfg.Timeout)/time.Second), "Timeout (in seconds) on requests to internal Keep services."+deprecated)
- flagset.StringVar(&cfg.ManagementToken, "management-token", cfg.ManagementToken, "Authorization token to be included in all health check requests.")
-
- var cfgPath string
- const defaultCfgPath = "/etc/arvados/keepproxy/keepproxy.yml"
- flagset.StringVar(&cfgPath, "config", defaultCfgPath, "Configuration file `path`")
- dumpConfig := flagset.Bool("dump-config", false, "write current configuration to stdout and exit")
- getVersion := flagset.Bool("version", false, "Print version information and exit.")
- flagset.Parse(os.Args[1:])
+ args = loader.MungeLegacyConfigArgs(logger, args[1:], "-legacy-keepproxy-config")
+ flags.Parse(args)
// Print version information if requested
if *getVersion {
fmt.Printf("keepproxy %s\n", version)
- return
+ return nil, nil
}
- err := config.LoadFile(cfg, cfgPath)
+ cfg, err := loader.Load()
if err != nil {
- h := os.Getenv("ARVADOS_API_HOST")
- t := os.Getenv("ARVADOS_API_TOKEN")
- if h == "" || t == "" || !os.IsNotExist(err) || cfgPath != defaultCfgPath {
- log.Fatal(err)
- }
- log.Print("DEPRECATED: No config file found, but ARVADOS_API_HOST and ARVADOS_API_TOKEN environment variables are set. Please use a config file instead.")
- cfg.Client.APIHost = h
- cfg.Client.AuthToken = t
- if regexp.MustCompile("^(?i:1|yes|true)$").MatchString(os.Getenv("ARVADOS_API_HOST_INSECURE")) {
- cfg.Client.Insecure = true
+ return nil, err
+ }
+ cluster, err := cfg.GetCluster("")
+ if err != nil {
+ return nil, err
+ }
+
+ if *dumpConfig {
+ out, err := yaml.Marshal(cfg)
+ if err != nil {
+ return nil, err
}
- if y, err := yaml.Marshal(cfg); err == nil && !*dumpConfig {
- log.Print("Current configuration:\n", string(y))
+ if _, err := os.Stdout.Write(out); err != nil {
+ return nil, err
}
- cfg.Timeout = arvados.Duration(time.Duration(*timeoutSeconds) * time.Second)
+ return nil, nil
}
+ return cluster, nil
+}
- if *dumpConfig {
- log.Fatal(config.DumpAndExit(cfg))
+func main() {
+ logger := log.New()
+ logger.Formatter = &log.JSONFormatter{
+ TimestampFormat: rfc3339NanoFixed,
+ }
+
+ cluster, err := configure(logger, os.Args)
+ if err != nil {
+ log.Fatal(err)
+ }
+ if cluster == nil {
+ return
}
log.Printf("keepproxy %s started", version)
- arv, err := arvadosclient.New(&cfg.Client)
+ if err := run(logger, cluster); err != nil {
+ log.Fatal(err)
+ }
+
+ log.Println("shutting down")
+}
+
+func run(logger log.FieldLogger, cluster *arvados.Cluster) error {
+ client, err := arvados.NewClientFromConfig(cluster)
+ if err != nil {
+ return err
+ }
+ client.AuthToken = cluster.SystemRootToken
+
+ arv, err := arvadosclient.New(client)
if err != nil {
- log.Fatalf("Error setting up arvados client %s", err.Error())
+ return fmt.Errorf("Error setting up arvados client %v", err)
}
- if cfg.Debug {
+ if cluster.SystemLogs.LogLevel == "debug" {
keepclient.DebugPrintf = log.Printf
}
kc, err := keepclient.MakeKeepClient(arv)
if err != nil {
- log.Fatalf("Error setting up keep client %s", err.Error())
+ return fmt.Errorf("Error setting up keep client %v", err)
}
keepclient.RefreshServiceDiscoveryOnSIGHUP()
- if cfg.PIDFile != "" {
- f, err := os.Create(cfg.PIDFile)
- if err != nil {
- log.Fatal(err)
- }
- defer f.Close()
- err = syscall.Flock(int(f.Fd()), syscall.LOCK_EX|syscall.LOCK_NB)
- if err != nil {
- log.Fatalf("flock(%s): %s", cfg.PIDFile, err)
- }
- defer os.Remove(cfg.PIDFile)
- err = f.Truncate(0)
- if err != nil {
- log.Fatalf("truncate(%s): %s", cfg.PIDFile, err)
- }
- _, err = fmt.Fprint(f, os.Getpid())
- if err != nil {
- log.Fatalf("write(%s): %s", cfg.PIDFile, err)
- }
- err = f.Sync()
- if err != nil {
- log.Fatalf("sync(%s): %s", cfg.PIDFile, err)
- }
+ if cluster.Collections.DefaultReplication > 0 {
+ kc.Want_replicas = cluster.Collections.DefaultReplication
}
- if cfg.DefaultReplicas > 0 {
- kc.Want_replicas = cfg.DefaultReplicas
+ var listen arvados.URL
+ for listen = range cluster.Services.Keepproxy.InternalURLs {
+ break
}
- listener, err = net.Listen("tcp", cfg.Listen)
- if err != nil {
- log.Fatalf("listen(%s): %s", cfg.Listen, err)
+ var lErr error
+ listener, lErr = net.Listen("tcp", listen.Host)
+ if lErr != nil {
+ return fmt.Errorf("listen(%s): %v", listen.Host, lErr)
}
+
if _, err := daemon.SdNotify(false, "READY=1"); err != nil {
log.Printf("Error notifying init daemon: %v", err)
}
- log.Println("Listening at", listener.Addr())
+ log.Println("listening at", listener.Addr())
// Shut down the server gracefully (by closing the listener)
// if SIGTERM is received.
signal.Notify(term, syscall.SIGINT)
// Start serving requests.
- router = MakeRESTRouter(!cfg.DisableGet, !cfg.DisablePut, kc, time.Duration(cfg.Timeout), cfg.ManagementToken)
- http.Serve(listener, httpserver.AddRequestIDs(httpserver.LogRequests(router)))
-
- log.Println("shutting down")
+ router = MakeRESTRouter(kc, time.Duration(cluster.API.KeepServiceRequestTimeout), cluster.SystemRootToken)
+ return http.Serve(listener, httpserver.AddRequestIDs(httpserver.LogRequests(router)))
}
type ApiTokenCache struct {
// MakeRESTRouter returns an http.Handler that passes GET and PUT
// requests to the appropriate handlers.
-func MakeRESTRouter(enable_get bool, enable_put bool, kc *keepclient.KeepClient, timeout time.Duration, mgmtToken string) http.Handler {
+func MakeRESTRouter(kc *keepclient.KeepClient, timeout time.Duration, mgmtToken string) http.Handler {
rest := mux.NewRouter()
transport := defaultTransport
},
}
- if enable_get {
- rest.HandleFunc(`/{locator:[0-9a-f]{32}\+.*}`, h.Get).Methods("GET", "HEAD")
- rest.HandleFunc(`/{locator:[0-9a-f]{32}}`, h.Get).Methods("GET", "HEAD")
+ rest.HandleFunc(`/{locator:[0-9a-f]{32}\+.*}`, h.Get).Methods("GET", "HEAD")
+ rest.HandleFunc(`/{locator:[0-9a-f]{32}}`, h.Get).Methods("GET", "HEAD")
- // List all blocks
- rest.HandleFunc(`/index`, h.Index).Methods("GET")
+ // List all blocks
+ rest.HandleFunc(`/index`, h.Index).Methods("GET")
- // List blocks whose hash has the given prefix
- rest.HandleFunc(`/index/{prefix:[0-9a-f]{0,32}}`, h.Index).Methods("GET")
- }
+ // List blocks whose hash has the given prefix
+ rest.HandleFunc(`/index/{prefix:[0-9a-f]{0,32}}`, h.Index).Methods("GET")
- if enable_put {
- rest.HandleFunc(`/{locator:[0-9a-f]{32}\+.*}`, h.Put).Methods("PUT")
- rest.HandleFunc(`/{locator:[0-9a-f]{32}}`, h.Put).Methods("PUT")
- rest.HandleFunc(`/`, h.Put).Methods("POST")
- rest.HandleFunc(`/{any}`, h.Options).Methods("OPTIONS")
- rest.HandleFunc(`/`, h.Options).Methods("OPTIONS")
- }
+ rest.HandleFunc(`/{locator:[0-9a-f]{32}\+.*}`, h.Put).Methods("PUT")
+ rest.HandleFunc(`/{locator:[0-9a-f]{32}}`, h.Put).Methods("PUT")
+ rest.HandleFunc(`/`, h.Put).Methods("POST")
+ rest.HandleFunc(`/{any}`, h.Options).Methods("OPTIONS")
+ rest.HandleFunc(`/`, h.Options).Methods("OPTIONS")
rest.Handle("/_health/{check}", &health.Handler{
Token: mgmtToken,
Description=Arvados Keep Proxy
Documentation=https://doc.arvados.org/
After=network.target
-AssertPathExists=/etc/arvados/keepproxy/keepproxy.yml
# systemd==229 (ubuntu:xenial) obeys StartLimitInterval in the [Unit] section
StartLimitInterval=0
"math/rand"
"net/http"
"net/http/httptest"
- "os"
"strings"
"sync"
"testing"
"time"
+ "git.curoverse.com/arvados.git/lib/config"
+ "git.curoverse.com/arvados.git/sdk/go/arvados"
"git.curoverse.com/arvados.git/sdk/go/arvadosclient"
"git.curoverse.com/arvados.git/sdk/go/arvadostest"
"git.curoverse.com/arvados.git/sdk/go/keepclient"
+ log "github.com/sirupsen/logrus"
. "gopkg.in/check.v1"
)
arvadostest.StopAPI()
}
-func runProxy(c *C, args []string, bogusClientToken bool) *keepclient.KeepClient {
- args = append([]string{"keepproxy"}, args...)
- os.Args = append(args, "-listen=:0")
+func runProxy(c *C, bogusClientToken bool) *keepclient.KeepClient {
+ cfg, err := config.NewLoader(nil, nil).Load()
+ c.Assert(err, Equals, nil)
+ cluster, err := cfg.GetCluster("")
+ c.Assert(err, Equals, nil)
+
+ cluster.Services.Keepproxy.InternalURLs = map[arvados.URL]arvados.ServiceInstance{arvados.URL{Host: ":0"}: arvados.ServiceInstance{}}
+
listener = nil
- go main()
+ go func() {
+ run(log.New(), cluster)
+ defer closeListener()
+ }()
waitForListener()
- arv, err := arvadosclient.MakeArvadosClient()
+ client := arvados.NewClientFromEnv()
+ arv, err := arvadosclient.New(client)
c.Assert(err, Equals, nil)
if bogusClientToken {
arv.ApiToken = "bogus-token"
}
func (s *ServerRequiredSuite) TestResponseViaHeader(c *C) {
- runProxy(c, nil, false)
+ runProxy(c, false)
defer closeListener()
req, err := http.NewRequest("POST",
}
func (s *ServerRequiredSuite) TestLoopDetection(c *C) {
- kc := runProxy(c, nil, false)
+ kc := runProxy(c, false)
defer closeListener()
sr := map[string]string{
}
func (s *ServerRequiredSuite) TestStorageClassesHeader(c *C) {
- kc := runProxy(c, nil, false)
+ kc := runProxy(c, false)
defer closeListener()
// Set up fake keepstore to record request headers
}
func (s *ServerRequiredSuite) TestDesiredReplicas(c *C) {
- kc := runProxy(c, nil, false)
+ kc := runProxy(c, false)
defer closeListener()
content := []byte("TestDesiredReplicas")
}
func (s *ServerRequiredSuite) TestPutWrongContentLength(c *C) {
- kc := runProxy(c, nil, false)
+ kc := runProxy(c, false)
defer closeListener()
content := []byte("TestPutWrongContentLength")
// fixes the invalid Content-Length header. In order to test
// our server behavior, we have to call the handler directly
// using an httptest.ResponseRecorder.
- rtr := MakeRESTRouter(true, true, kc, 10*time.Second, "")
+ rtr := MakeRESTRouter(kc, 10*time.Second, "")
type testcase struct {
sendLength string
}
func (s *ServerRequiredSuite) TestManyFailedPuts(c *C) {
- kc := runProxy(c, nil, false)
+ kc := runProxy(c, false)
defer closeListener()
router.(*proxyHandler).timeout = time.Nanosecond
}
func (s *ServerRequiredSuite) TestPutAskGet(c *C) {
- kc := runProxy(c, nil, false)
+ kc := runProxy(c, false)
defer closeListener()
hash := fmt.Sprintf("%x", md5.Sum([]byte("foo")))
}
func (s *ServerRequiredSuite) TestPutAskGetForbidden(c *C) {
- kc := runProxy(c, nil, true)
+ kc := runProxy(c, true)
defer closeListener()
hash := fmt.Sprintf("%x+3", md5.Sum([]byte("bar")))
}
-func (s *ServerRequiredSuite) TestGetDisabled(c *C) {
- kc := runProxy(c, []string{"-no-get"}, false)
- defer closeListener()
-
- hash := fmt.Sprintf("%x", md5.Sum([]byte("baz")))
-
- {
- _, _, err := kc.Ask(hash)
- errNotFound, _ := err.(keepclient.ErrNotFound)
- c.Check(errNotFound, NotNil)
- c.Assert(err, ErrorMatches, `.*HTTP 405.*`)
- c.Log("Ask 1")
- }
-
- {
- hash2, rep, err := kc.PutB([]byte("baz"))
- c.Check(hash2, Matches, fmt.Sprintf(`^%s\+3(\+.+)?$`, hash))
- c.Check(rep, Equals, 2)
- c.Check(err, Equals, nil)
- c.Log("PutB")
- }
-
- {
- blocklen, _, err := kc.Ask(hash)
- errNotFound, _ := err.(keepclient.ErrNotFound)
- c.Check(errNotFound, NotNil)
- c.Assert(err, ErrorMatches, `.*HTTP 405.*`)
- c.Check(blocklen, Equals, int64(0))
- c.Log("Ask 2")
- }
-
- {
- _, blocklen, _, err := kc.Get(hash)
- errNotFound, _ := err.(keepclient.ErrNotFound)
- c.Check(errNotFound, NotNil)
- c.Assert(err, ErrorMatches, `.*HTTP 405.*`)
- c.Check(blocklen, Equals, int64(0))
- c.Log("Get")
- }
-}
-
-func (s *ServerRequiredSuite) TestPutDisabled(c *C) {
- kc := runProxy(c, []string{"-no-put"}, false)
- defer closeListener()
-
- hash2, rep, err := kc.PutB([]byte("quux"))
- c.Check(hash2, Equals, "")
- c.Check(rep, Equals, 0)
- c.Check(err, FitsTypeOf, keepclient.InsufficientReplicasError(errors.New("")))
-}
-
func (s *ServerRequiredSuite) TestCorsHeaders(c *C) {
- runProxy(c, nil, false)
+ runProxy(c, false)
defer closeListener()
{
}
func (s *ServerRequiredSuite) TestPostWithoutHash(c *C) {
- runProxy(c, nil, false)
+ runProxy(c, false)
defer closeListener()
{
// With a valid but non-existing prefix (expect "\n")
// With an invalid prefix (expect error)
func (s *ServerRequiredSuite) TestGetIndex(c *C) {
- kc := runProxy(c, nil, false)
+ kc := runProxy(c, false)
defer closeListener()
// Put "index-data" blocks
}
func (s *ServerRequiredSuite) TestCollectionSharingToken(c *C) {
- kc := runProxy(c, nil, false)
+ kc := runProxy(c, false)
defer closeListener()
hash, _, err := kc.PutB([]byte("shareddata"))
c.Check(err, IsNil)
}
func (s *ServerRequiredSuite) TestPutAskGetInvalidToken(c *C) {
- kc := runProxy(c, nil, false)
+ kc := runProxy(c, false)
defer closeListener()
// Put a test block
}
func (s *ServerRequiredSuite) TestAskGetKeepProxyConnectionError(c *C) {
- kc := runProxy(c, nil, false)
+ kc := runProxy(c, false)
defer closeListener()
// Point keepproxy at a non-existent keepstore
}
func (s *NoKeepServerSuite) TestAskGetNoKeepServerError(c *C) {
- kc := runProxy(c, nil, false)
+ kc := runProxy(c, false)
defer closeListener()
hash := fmt.Sprintf("%x", md5.Sum([]byte("foo")))
}
func (s *ServerRequiredSuite) TestPing(c *C) {
- kc := runProxy(c, nil, false)
+ kc := runProxy(c, false)
defer closeListener()
- rtr := MakeRESTRouter(true, true, kc, 10*time.Second, arvadostest.ManagementToken)
+ rtr := MakeRESTRouter(kc, 10*time.Second, arvadostest.ManagementToken)
req, err := http.NewRequest("GET",
"http://"+listener.Addr().String()+"/_health/ping",
+++ /dev/null
-// Copyright (C) The Arvados Authors. All rights reserved.
-//
-// SPDX-License-Identifier: AGPL-3.0
-
-package main
-
-import (
- "encoding/json"
- "flag"
- "fmt"
- "os"
-)
-
-func usage() {
- c := DefaultConfig()
- c.Client.APIHost = "zzzzz.arvadosapi.com:443"
- exampleConfigFile, err := json.MarshalIndent(c, " ", " ")
- if err != nil {
- panic(err)
- }
- fmt.Fprintf(os.Stderr, `
-
-Keepproxy forwards GET and PUT requests to keepstore servers. See
-http://doc.arvados.org/install/install-keepproxy.html
-
-Usage: keepproxy [-config path/to/keepproxy.yml]
-
-Options:
-`)
- flag.PrintDefaults()
- fmt.Fprintf(os.Stderr, `
-Example config file:
- %s
-
-Client.APIHost:
-
- Address (or address:port) of the Arvados API endpoint.
-
-Client.AuthToken:
-
- Anonymous API token.
-
-Client.Insecure:
-
- True if your Arvados API endpoint uses an unverifiable SSL/TLS
- certificate.
-
-Listen:
-
- Local port to listen on. Can be "address:port" or ":port", where
- "address" is a host IP address or name and "port" is a port number
- or name.
-
-DisableGet:
-
- Respond 404 to GET and HEAD requests.
-
-DisablePut:
-
- Respond 404 to PUT, POST, and OPTIONS requests.
-
-DefaultReplicas:
-
- Default number of replicas to write if not specified by the
- client. If this is zero or omitted, the site-wide
- defaultCollectionReplication configuration will be used.
-
-Timeout:
-
- Timeout for requests to keep services, with units (e.g., "120s",
- "2m").
-
-PIDFile:
-
- Path to PID file. During startup this file will be created if
- needed, and locked using flock() until keepproxy exits. If it is
- already locked, or any error is encountered while writing to it,
- keepproxy will exit immediately. If omitted or empty, no PID file
- will be used.
-
-Debug:
-
- Enable debug logging.
-
-ManagementToken:
-
- Authorization token to be included in all health check requests.
-
-`, exampleConfigFile)
-}
ExternalURL: "https://$localip:${services[workbench2-ssl]}"
SSO:
ExternalURL: "https://$localip:${services[sso]}"
+ Keepproxy:
+ InternalURLs:
+ "http://localhost:${services[keepproxy]}/": {}
+ ExternalURL: "http://$localip:${services[keepproxy-ssl]}/"
Websocket:
ExternalURL: "wss://$localip:${services[websockets-ssl]}/websocket"
InternalURLs:
GitSSH:
ExternalURL: "ssh://git@$localip:"
GitHTTP:
- ExternalURL: "http://$localip:${services[arv-git-httpd]}/"
+ InternalURLs:
+ "http://localhost:${services[arv-git-httpd]}/": {}
+ ExternalURL: "https://$localip:${services[arv-git-httpd-ssl]}/"
WebDAV:
InternalURLs:
"http://localhost:${services[keep-web]}/": {}
Workbench:
SecretKeyBase: $workbench_secret_key_base
ArvadosDocsite: http://$localip:${services[doc]}/
+ Git:
+ GitCommand: /usr/share/gitolite3/gitolite-shell
+ GitoliteHome: /var/lib/arvados/git
+ Repositories: /var/lib/arvados/git/repositories
EOF
/usr/local/lib/arvbox/yml_override.py /var/lib/arvados/cluster_config.yml
[controller-ssl]=8000
[sso]=8900
[composer]=4200
+ [arv-git-httpd-ssl]=9000
[arv-git-httpd]=9001
[keep-web]=9003
[keep-web-ssl]=9002
export PATH="$PATH:/var/lib/arvados/git/bin"
cd ~git
-exec /usr/local/bin/arv-git-httpd \
- -address=:${services[arv-git-httpd]} \
- -git-command=/usr/share/gitolite3/gitolite-shell \
- -gitolite-home=/var/lib/arvados/git \
- -repo-root=/var/lib/arvados/git/repositories
+exec /usr/local/bin/arv-git-httpd
echo $UUID > /var/lib/arvados/keepproxy-uuid
fi
-exec /usr/local/bin/keepproxy -listen=:${services[keepproxy]}
+exec /usr/local/bin/keepproxy
}
}
+ upstream arvados-git-httpd {
+ server localhost:${services[arv-git-httpd]};
+ }
+ server {
+ listen *:${services[arv-git-httpd-ssl]} ssl default_server;
+ server_name arvados-git-httpd;
+ proxy_connect_timeout 90s;
+ proxy_read_timeout 300s;
+
+ ssl on;
+ ssl_certificate "${server_cert}";
+ ssl_certificate_key "${server_cert_key}";
+ client_max_body_size 50m;
+
+ location / {
+ proxy_pass http://arvados-git-httpd;
+ proxy_set_header Host \$http_host;
+ proxy_set_header X-Forwarded-For \$proxy_add_x_forwarded_for;
+ proxy_set_header X-Forwarded-Proto https;
+ proxy_redirect off;
+ }
+ }
+
}
EOF
"ignore": "test",
"package": [
{
- "checksumSHA1": "j4je0EzPGzjb6INLY1BHZ+hyMjc=",
+ "checksumSHA1": "jfYWZyRWLMfG0J5K7G2K8a9AKfs=",
"origin": "github.com/curoverse/goamz/aws",
"path": "github.com/AdRoll/goamz/aws",
- "revision": "888b4804f2653cd35ebcc95f046079e63b5b2799",
- "revisionTime": "2017-07-27T13:52:37Z"
+ "revision": "1bba09f407ef1d02c90bc37eff7e91e2231fa587",
+ "revisionTime": "2019-09-05T14:15:25Z"
},
{
- "checksumSHA1": "0+n3cT6e7sQCCbBAH8zg6neiHTk=",
+ "checksumSHA1": "lqoARtBgwnvhEhLyIjR3GLnR5/c=",
"origin": "github.com/curoverse/goamz/s3",
"path": "github.com/AdRoll/goamz/s3",
- "revision": "888b4804f2653cd35ebcc95f046079e63b5b2799",
- "revisionTime": "2017-07-27T13:52:37Z"
+ "revision": "1bba09f407ef1d02c90bc37eff7e91e2231fa587",
+ "revisionTime": "2019-09-05T14:15:25Z"
},
{
"checksumSHA1": "tvxbsTkdjB0C/uxEglqD6JfVnMg=",
"origin": "github.com/curoverse/goamz/s3/s3test",
"path": "github.com/AdRoll/goamz/s3/s3test",
- "revision": "888b4804f2653cd35ebcc95f046079e63b5b2799",
- "revisionTime": "2017-07-27T13:52:37Z"
+ "revision": "1bba09f407ef1d02c90bc37eff7e91e2231fa587",
+ "revisionTime": "2019-09-05T14:15:25Z"
},
{
"checksumSHA1": "KF4DsRUpZ+h+qRQ/umRAQZfVvw0=",
projects / arvados.git / commitdiff