From: Tom Clegg Date: Wed, 1 Apr 2015 22:34:38 +0000 (-0400) Subject: 5416: Browse git repository contents in workbench. X-Git-Tag: 1.1.0~1664^2~7 X-Git-Url: https://git.arvados.org/arvados.git/commitdiff_plain/9136a1b1314084e149f86ceec16d1482ccf5d8af?ds=inline 5416: Browse git repository contents in workbench. --- diff --git a/apps/workbench/app/controllers/repositories_controller.rb b/apps/workbench/app/controllers/repositories_controller.rb index d32c92a1e7..c5b3501b32 100644 --- a/apps/workbench/app/controllers/repositories_controller.rb +++ b/apps/workbench/app/controllers/repositories_controller.rb @@ -16,4 +16,20 @@ class RepositoriesController < ApplicationController panes.delete('Attributes') if !current_user.is_admin panes end + + def show_tree + @commit = params[:commit] + @path = params[:path] || '' + @subtree = @object.ls_subtree @commit, @path.chomp('/') + end + + def show_blob + @commit = params[:commit] + @path = params[:path] + @blobdata = @object.cat_file @commit, @path + end + + def show_commit + @commit = params[:commit] + end end diff --git a/apps/workbench/app/models/repository.rb b/apps/workbench/app/models/repository.rb index b062dda861..48c7f9efcb 100644 --- a/apps/workbench/app/models/repository.rb +++ b/apps/workbench/app/models/repository.rb @@ -12,4 +12,105 @@ class Repository < ArvadosBase [] end end + + def show commit_sha1 + refresh + run_git 'show', commit_sha1 + end + + def cat_file commit_sha1, path + refresh + run_git 'cat-file', 'blob', commit_sha1 + ':' + path + end + + def ls_tree_lr commit_sha1 + refresh + run_git 'ls-tree', '-l', '-r', commit_sha1 + end + + # subtree returns a list of files under the given path at the + # specified commit. Results are returned as an array of file nodes, + # where each file node is an array [file mode, blob sha1, file size + # in bytes, path relative to the given directory]. If the path is + # not found, [] is returned. + def ls_subtree commit, path + path = path.chomp '/' + subtree = [] + ls_tree_lr(commit).each_line do |line| + mode, type, sha1, size, filepath = line.split + next if type != 'blob' + if filepath[0,path.length] == path and + (path == '' or filepath[path.length] == '/') + subtree << [mode.to_i(8), sha1, size.to_i, + filepath[path.length,filepath.length]] + end + end + subtree + end + + protected + + # refresh fetches the latest repository content into the local + # cache. It is a no-op if it has already been run on this object: + # this (pretty much) avoids doing more than one remote git operation + # per Workbench request. + def refresh + run_git 'fetch', http_fetch_url, '+*:*' unless @fresh + @fresh = true + end + + # http_fetch_url returns an http:// or https:// fetch-url which can + # accept arvados API token authentication. The API server currently + # advertises SSH fetch-urls, which work for users with SSH keys but + # are useless for fetching repository content into workbench itself. + def http_fetch_url + "https://git.#{uuid[0,5]}.arvadosapi.com/#{name}.git" + end + + # run_git sets up the ARVADOS_API_TOKEN environment variable, + # creates a local git directory for this repository if necessary, + # executes "git --git-dir localgitdir {args to run_git}", and + # returns the output. It raises GitCommandError if git exits + # non-zero. + def run_git *gitcmd + if not @workdir + workdir = File.expand_path uuid+'.git', Rails.configuration.repository_cache + if not File.exists? workdir + FileUtils.mkdir_p Rails.configuration.repository_cache + [['git', 'init', '--bare', workdir], + ].each do |cmd| + system *cmd + raise GitCommandError.new($?.to_s) unless $?.exitstatus == 0 + end + end + @workdir = workdir + end + [['git', '--git-dir', @workdir, 'config', '--local', + "credential.#{http_fetch_url}.username", 'none'], + ['git', '--git-dir', @workdir, 'config', '--local', + "credential.#{http_fetch_url}.helper", + '!token(){ echo password="$ARVADOS_API_TOKEN"; }; token'], + ['git', '--git-dir', @workdir, 'config', '--local', + 'http.sslVerify', + Rails.configuration.arvados_insecure_https ? 'false' : 'true'], + ].each do |cmd| + system *cmd + raise GitCommandError.new($?.to_s) unless $?.exitstatus == 0 + end + env = {}. + merge(ENV). + merge('ARVADOS_API_TOKEN' => Thread.current[:arvados_api_token]) + cmd = ['git', '--git-dir', @workdir] + gitcmd + io = IO.popen(env, cmd, err: [:child, :out]) + output = io.read + io.close + # "If [io] is opened by IO.popen, close sets $?." --ruby 2.2.1 docs + unless $?.exitstatus == 0 + raise GitCommandError.new("`git #{gitcmd.join ' '}` #{$?}: #{output}") + end + output + end + + class GitCommandError < StandardError + end end diff --git a/apps/workbench/app/views/pipeline_instances/_running_component.html.erb b/apps/workbench/app/views/pipeline_instances/_running_component.html.erb index 1a9cb3562a..cc3b4c83d5 100644 --- a/apps/workbench/app/views/pipeline_instances/_running_component.html.erb +++ b/apps/workbench/app/views/pipeline_instances/_running_component.html.erb @@ -96,6 +96,12 @@
+ <% # link to repo tree/file only if the repo is readable + # and the commit is a sha1 + repo = + (/^[0-9a-f]{40}$/ =~ current_component[:script_version] and + Repository.where(name: current_component[:repository]).first) + %> <% [:script, :repository, :script_version, :supplied_script_version, :nondeterministic].each do |k| %>
@@ -104,6 +110,12 @@ <% if current_component[k].nil? %> (none) + <% elsif repo and k == :repository %> + <%= link_to current_component[k], show_repository_tree_path(id: repo.uuid, commit: current_component[:script_version], path: '/') %> + <% elsif repo and k == :script %> + <%= link_to current_component[k], show_repository_blob_path(id: repo.uuid, commit: current_component[:script_version], path: 'crunch_scripts/'+current_component[:script]) %> + <% elsif repo and k == :script_version %> + <%= link_to current_component[k], show_repository_commit_path(id: repo.uuid, commit: current_component[:script_version]) %> <% else %> <%= current_component[k] %> <% end %> diff --git a/apps/workbench/app/views/repositories/_repository_breadcrumbs.html.erb b/apps/workbench/app/views/repositories/_repository_breadcrumbs.html.erb new file mode 100644 index 0000000000..736c1873e5 --- /dev/null +++ b/apps/workbench/app/views/repositories/_repository_breadcrumbs.html.erb @@ -0,0 +1,13 @@ +
+ Browsing <%= @object.name %> repository at commit + <%= link_to(@commit, show_repository_commit_path(id: @object.uuid, commit: @commit), title: 'show commit message') %> +
+

+ <%= link_to(@object.name, show_repository_tree_path(id: @object.uuid, commit: @commit, path: '/'), title: 'show root directory of source tree') %> + <% parents = '' + (@path || '').split('/').each do |pathpart| + parents = parents + pathpart + '/' + %> + / <%= link_to pathpart, show_repository_tree_path(id: @object.uuid, commit: @commit, path: parents) %> + <% end %> +

diff --git a/apps/workbench/app/views/repositories/show_blob.html.erb b/apps/workbench/app/views/repositories/show_blob.html.erb new file mode 100644 index 0000000000..acc34d116e --- /dev/null +++ b/apps/workbench/app/views/repositories/show_blob.html.erb @@ -0,0 +1,13 @@ +<%= render partial: 'repository_breadcrumbs' %> + +<% if not @blobdata.valid_encoding? %> +
+

+ This file has an invalid text encoding, so it can't be shown + here. (This probably just means it's a binary file, not a text + file.) +

+
+<% else %> + 
<%= @blobdata %>
+<% end %> diff --git a/apps/workbench/app/views/repositories/show_commit.html.erb b/apps/workbench/app/views/repositories/show_commit.html.erb new file mode 100644 index 0000000000..3690be6996 --- /dev/null +++ b/apps/workbench/app/views/repositories/show_commit.html.erb @@ -0,0 +1,3 @@ +<%= render partial: 'repository_breadcrumbs' %> + +
<%= @object.show @commit %>
diff --git a/apps/workbench/app/views/repositories/show_tree.html.erb b/apps/workbench/app/views/repositories/show_tree.html.erb new file mode 100644 index 0000000000..4e2fcece95 --- /dev/null +++ b/apps/workbench/app/views/repositories/show_tree.html.erb @@ -0,0 +1,40 @@ +<%= render partial: 'repository_breadcrumbs' %> + + + + + + + + + + <% @subtree.each do |mode, sha1, size, subpath| %> + + + + + <% end %> + <% if @subtree.empty? %> + + + + <% end %> + + +
FileSize
+ + <% pathparts = subpath.sub(/^\//, '').split('/') + basename = pathparts.pop + parents = @path + pathparts.each do |pathpart| %> + <% parents = parents + '/' + pathpart %> + <%= link_to pathpart, url_for(path: parents) %> + / + <% end %> + + <%= link_to basename, url_for(action: :show_blob, path: parents + '/' + basename) %> + + <%= human_readable_bytes_html(size) %> +
+ No files found. +
diff --git a/apps/workbench/config/application.default.yml b/apps/workbench/config/application.default.yml index 8aeacf4b61..4061ee83d8 100644 --- a/apps/workbench/config/application.default.yml +++ b/apps/workbench/config/application.default.yml @@ -139,8 +139,14 @@ common: default_openid_prefix: https://www.google.com/accounts/o8/id send_user_setup_notification_email: true - # Set user_profile_form_fields to enable and configure the user profile page. - # Default is set to false. A commented setting with full description is provided below. + # Scratch directory used by the remote repository browsing + # feature. If it doesn't exist, it (and any missing parents) will be + # created using mkdir_p. + repository_cache: <%= File.expand_path 'tmp/git', Rails.root %> + + # Set user_profile_form_fields to enable and configure the user + # profile page. Default is set to false. A commented example with + # full description is provided below. user_profile_form_fields: false # Below is a sample setting of user_profile_form_fields config parameter. diff --git a/apps/workbench/config/routes.rb b/apps/workbench/config/routes.rb index 7ed02e7dc9..44d7ded6c1 100644 --- a/apps/workbench/config/routes.rb +++ b/apps/workbench/config/routes.rb @@ -26,6 +26,11 @@ ArvadosWorkbench::Application.routes.draw do resources :repositories do post 'share_with', on: :member end + # {format: false} prevents rails from treating "foo.png" as foo?format=png + get '/repositories/:id/tree/:commit' => 'repositories#show_tree' + get '/repositories/:id/tree/:commit/*path' => 'repositories#show_tree', as: :show_repository_tree, format: false + get '/repositories/:id/blob/:commit/*path' => 'repositories#show_blob', as: :show_repository_blob, format: false + get '/repositories/:id/commit/:commit' => 'repositories#show_commit', as: :show_repository_commit match '/logout' => 'sessions#destroy', via: [:get, :post] get '/logged_out' => 'sessions#index' resources :users do diff --git a/apps/workbench/test/controllers/repositories_controller_test.rb b/apps/workbench/test/controllers/repositories_controller_test.rb index f95bb7731f..25bf557685 100644 --- a/apps/workbench/test/controllers/repositories_controller_test.rb +++ b/apps/workbench/test/controllers/repositories_controller_test.rb @@ -1,7 +1,9 @@ require 'test_helper' +require 'helpers/repository_stub_helper' require 'helpers/share_object_helper' class RepositoriesControllerTest < ActionController::TestCase + include RepositoryStubHelper include ShareObjectHelper [ @@ -62,4 +64,61 @@ class RepositoriesControllerTest < ActionController::TestCase end end end + + ### Browse repository content + + [:active, :spectator].each do |user| + test "show tree to #{user}" do + reset_api_fixtures_after_test false + sha1, _, _ = stub_repo_content + get :show_tree, { + id: api_fixture('repositories')['foo']['uuid'], + commit: sha1, + }, session_for(user) + assert_response :success + assert_select 'tr td a', 'COPYING' + assert_select 'tr td', '625 bytes' + assert_select 'tr td a', 'apps' + assert_select 'tr td a', 'workbench' + assert_select 'tr td a', 'Gemfile' + assert_select 'tr td', '33.7 KiB' + end + + test "show commit to #{user}" do + reset_api_fixtures_after_test false + sha1, commit, _ = stub_repo_content + get :show_commit, { + id: api_fixture('repositories')['foo']['uuid'], + commit: sha1, + }, session_for(user) + assert_response :success + assert_select 'pre', h(commit) + end + + test "show blob to #{user}" do + reset_api_fixtures_after_test false + sha1, _, filedata = stub_repo_content filename: 'COPYING' + get :show_blob, { + id: api_fixture('repositories')['foo']['uuid'], + commit: sha1, + path: 'COPYING', + }, session_for(user) + assert_response :success + assert_select 'pre', h(filedata) + end + end + + ['', '/'].each do |path| + test "show tree with path '#{path}'" do + reset_api_fixtures_after_test false + sha1, _, _ = stub_repo_content filename: 'COPYING' + get :show_tree, { + id: api_fixture('repositories')['foo']['uuid'], + commit: sha1, + path: path, + }, session_for(:active) + assert_response :success + assert_select 'tr td', 'COPYING' + end + end end diff --git a/apps/workbench/test/helpers/repository_stub_helper.rb b/apps/workbench/test/helpers/repository_stub_helper.rb new file mode 100644 index 0000000000..b7d0573b85 --- /dev/null +++ b/apps/workbench/test/helpers/repository_stub_helper.rb @@ -0,0 +1,33 @@ +module RepositoryStubHelper + # Supply some fake git content. + def stub_repo_content opts={} + fakesha1 = opts[:sha1] || 'abcdefabcdefabcdefabcdefabcdefabcdefabcd' + fakefilename = opts[:filename] || 'COPYING' + fakefilesrc = File.expand_path('../../../../../'+fakefilename, __FILE__) + fakefile = File.read fakefilesrc + fakecommit = <<-EOS + commit abcdefabcdefabcdefabcdefabcdefabcdefabcd + Author: Fake R + Date: Wed Apr 1 11:59:59 2015 -0400 + + It's a fake commit. + + EOS + Repository.any_instance.stubs(:ls_tree_lr).with(fakesha1).returns <<-EOS + 100644 blob eec475862e6ec2a87554e0fca90697e87f441bf5 226 .gitignore + 100644 blob acbd7523ed49f01217874965aa3180cccec89d61 625 COPYING + 100644 blob d645695673349e3947e8e5ae42332d0ac3164cd7 11358 LICENSE-2.0.txt + 100644 blob c7a36c355b4a2b94dfab45c9748330022a788c91 622 README + 100644 blob dba13ed2ddf783ee8118c6a581dbf75305f816a3 34520 agpl-3.0.txt + 100644 blob 9bef02bbfda670595750fd99a4461005ce5b8f12 695 apps/workbench/.gitignore + 100644 blob b51f674d90f68bfb50d9304068f915e42b04aea4 2249 apps/workbench/Gemfile + 100644 blob b51f674d90f68bfb50d9304068f915e42b04aea4 2249 apps/workbench/Gemfile + 100755 blob cdd5ebaff27781f93ab85e484410c0ce9e97770f 1012 crunch_scripts/hash + EOS + Repository.any_instance. + stubs(:cat_file).with(fakesha1, fakefilename).returns fakefile + Repository.any_instance. + stubs(:show).with(fakesha1).returns fakecommit + return fakesha1, fakecommit, fakefile + end +end diff --git a/apps/workbench/test/integration/repositories_browse_test.rb b/apps/workbench/test/integration/repositories_browse_test.rb new file mode 100644 index 0000000000..147bf464be --- /dev/null +++ b/apps/workbench/test/integration/repositories_browse_test.rb @@ -0,0 +1,37 @@ +require 'integration_helper' +require 'helpers/repository_stub_helper' +require 'helpers/share_object_helper' + +class RepositoriesTest < ActionDispatch::IntegrationTest + include RepositoryStubHelper + include ShareObjectHelper + + reset_api_fixtures :after_each_test, false + + setup do + need_javascript + end + + test "browse repository from jobs#show" do + sha1 = api_fixture('jobs')['running']['script_version'] + _, fakecommit, fakefile = + stub_repo_content sha1: sha1, filename: 'crunch_scripts/hash' + show_object_using 'active', 'jobs', 'running', sha1 + click_on api_fixture('jobs')['running']['script'] + assert_text fakefile + click_on 'crunch_scripts' + assert_selector 'td a', text: 'hash' + click_on 'foo' + assert_selector 'td a', text: 'crunch_scripts' + click_on sha1 + assert_text fakecommit + + show_object_using 'active', 'jobs', 'running', sha1 + click_on 'active/foo' + assert_selector 'td a', text: 'crunch_scripts' + + show_object_using 'active', 'jobs', 'running', sha1 + click_on sha1 + assert_text fakecommit + end +end diff --git a/apps/workbench/test/test_helper.rb b/apps/workbench/test/test_helper.rb index ade6292efe..630a64f289 100644 --- a/apps/workbench/test/test_helper.rb +++ b/apps/workbench/test/test_helper.rb @@ -270,12 +270,17 @@ class ActiveSupport::TestCase end def after_teardown - if self.class.want_reset_api_fixtures[:after_each_test] + if self.class.want_reset_api_fixtures[:after_each_test] and + @want_reset_api_fixtures != false self.class.reset_api_fixtures_now end super end + def reset_api_fixtures_after_test t=true + @want_reset_api_fixtures = t + end + protected def self.reset_api_fixtures_now # Never try to reset fixtures when we're just using test diff --git a/services/api/test/fixtures/jobs.yml b/services/api/test/fixtures/jobs.yml index c662062704..23b32ef9cd 100644 --- a/services/api/test/fixtures/jobs.yml +++ b/services/api/test/fixtures/jobs.yml @@ -7,6 +7,8 @@ running: created_at: <%= 3.minute.ago.to_s(:db) %> started_at: <%= 3.minute.ago.to_s(:db) %> finished_at: ~ + script: hash + repository: active/foo script_version: 1de84a854e2b440dc53bf42f8548afa4c17da332 running: true success: ~ @@ -31,6 +33,8 @@ running_cancelled: created_at: <%= 4.minute.ago.to_s(:db) %> started_at: <%= 3.minute.ago.to_s(:db) %> finished_at: ~ + script: hash + repository: active/foo script_version: 1de84a854e2b440dc53bf42f8548afa4c17da332 running: true success: ~ @@ -56,6 +60,8 @@ uses_nonexistent_script_version: created_at: <%= 5.minute.ago.to_s(:db) %> started_at: <%= 3.minute.ago.to_s(:db) %> finished_at: <%= 2.minute.ago.to_s(:db) %> + script: hash + repository: active/foo running: false success: true output: d41d8cd98f00b204e9800998ecf8427e+0