X-Git-Url: https://git.arvados.org/arvados.git/blobdiff_plain/14a6eb786a0d01e86ccae7645e880661caf6f0cb..810846168f8e14f63caefac534b843b7681b881f:/doc/Rakefile

diff --git a/doc/Rakefile b/doc/Rakefile
index 63dc16d25d..3ec494f57d 100644
--- a/doc/Rakefile
+++ b/doc/Rakefile
@@ -8,12 +8,116 @@
 # generating the documentation for the SDKs, which (the R docs
 # especially) take a fair bit of time and slow down the edit-preview
 # cycle.
+#
+# To generate and view the documentation locally, run this command
+#
+#   rake && sensible-browser .site/index.html
+#
+# Or alternatively:
+#
+#   baseurl=http://localhost:8000 rake && rake run
+#
+# and then visit http://localhost:8000 in a browser.
+
+require "uri"
 
 require "rubygems"
 require "colorize"
 
-task :generate => [ :realclean, 'sdk/python/arvados/index.html', 'sdk/R/arvados/index.html', 'sdk/java-v2/javadoc/index.html' ] do
+def can_run?(*command, **options)
+  options = {
+    :in => :close,
+    :out => [File::NULL, "w"],
+  }.merge(options)
+  system(*command, **options)
+end
+
+class JavaSDK
+  def self.build_path
+    "sdk/java-v2"
+  end
+
+  def self.can_build?
+    can_run?("gradle", "--version")
+  end
+end
+
+class PythonSDK
+  def self.build_path
+    "sdk/python/arvados"
+  end
+
+  def self.can_build?
+    can_run?("./pysdk_pdoc.py", "--version")
+  end
+end
+
+class RSDK
+  def self.build_path
+    "sdk/R"
+  end
+
+  def self.can_build?
+    can_run?("R", "--version")
+  end
+end
+
+$build_sdks = begin
+  no_sdk_env = ENV.fetch("NO_SDK", "")
+  sdks_env = ENV.fetch("sdks", "")
+  all_sdks = Hash[[JavaSDK, PythonSDK, RSDK].map { |c| [c.name, c] }]
+
+  if no_sdk_env != "" and sdks_env != ""
+    fail "both NO_SDK and sdks defined in environment"
+  elsif sdks_env != ""
+    # Nothing to do
+  elsif no_sdk_env != "" or File.exist?("no-sdk")
+    sdks_env = "none"
+  end
+
+  if sdks_env == ""
+    all_sdks.each_pair.filter_map do |name, sdk|
+      if sdk.can_build?
+        sdk
+      else
+        puts "Warning: cannot build #{name.replace(/SDK$/, ' SDK')} documentation, skipping".colorize(:light_red)
+      end
+    end
+  else
+    wanted_sdks = []
+    sdks_env.split(/\s*[,\s]\s*/).each do |key|
+      key = "#{key.capitalize}SDK"
+      if key == "AllSDK"
+        wanted_sdks = all_sdks.values
+      elsif key == "NoneSDK"
+        wanted_sdks.clear
+      elsif sdk = all_sdks[key]
+        wanted_sdks << sdk
+      else
+        fail "cannot build documentation for unknown #{key}"
+      end
+    end
+    wanted_sdks
+  end
+end
+
+module Zenweb
+  class Site
+    @binary_files = %w[png jpg gif eot svg ttf woff2? ico pdf m4a t?gz xlsx]
+  end
+end
+
+task :generate => [ :realclean, 'sdk/python/arvados.html', 'sdk/R/arvados/index.html', 'sdk/java-v2/javadoc/index.html' ] do
   vars = ['baseurl', 'arvados_cluster_uuid', 'arvados_api_host', 'arvados_workbench_host']
+  if ! ENV.key?('baseurl') || ENV['baseurl'] == ""
+    if !ENV.key?('WORKSPACE') || ENV['WORKSPACE'] == ""
+      puts "The `baseurl` variable was not specified and the `WORKSPACE` environment variable is not set. Defaulting `baseurl` to file://#{pwd}/.site"
+      ENV['baseurl'] = "file://#{pwd}/.site/"
+    else
+      puts "The `baseurl` variable was not specified, defaulting to a value derived from the `WORKSPACE` environment variable"
+      ENV['baseurl'] = "file://#{ENV['WORKSPACE']}/doc/.site/"
+    end
+  end
   vars.each do |v|
     if ENV[v]
       website.config.h[v] = ENV[v]
@@ -21,42 +125,38 @@ task :generate => [ :realclean, 'sdk/python/arvados/index.html', 'sdk/R/arvados/
   end
 end
 
-file "sdk/python/arvados/index.html" do |t|
-  if File.exists? "no-sdk"
-    next
-  end
-  `which epydoc`
-  if $? == 0
-    STDERR.puts `epydoc --html --parse-only -o sdk/python/arvados ../sdk/python/arvados/ 2>&1`
-    raise if $? != 0
-  else
-    puts "Warning: epydoc not found, Python documentation will not be generated".colorize(:light_red)
-  end
+file ["install/new_cluster_checklist_Azure.xlsx", "install/new_cluster_checklist_AWS.xlsx"] do |t|
+  cp(t, t)
+end
+
+file "sdk/python/arvados.html" do |t|
+  next unless $build_sdks.include?(PythonSDK)
+  raise unless system("python3", "setup.py", "build",
+                      chdir: "../sdk/python", out: :err)
+  raise unless system("python3", "pysdk_pdoc.py",
+                      out: :err)
 end
 
 file "sdk/R/arvados/index.html" do |t|
-  if File.exists? "no-sdk"
-    next
-  end
-  `which R`
-  if $? == 0
-    tgt = Dir.pwd
-    Dir.mkdir("sdk/R")
-    Dir.mkdir("sdk/R/arvados")
-    docfiles = []
-    Dir.chdir("../sdk/R/") do
-      STDERR.puts `Rscript createDoc.R README.Rmd #{tgt}/sdk/R/README.md 2>&1`
-      Dir.entries("man").each do |rd|
-        if rd[-3..-1] == ".Rd"
-          htmlfile = "#{rd[0..-4]}.html"
-          `R CMD Rdconv -t html man/#{rd} > #{tgt}/sdk/R/arvados/#{htmlfile}`
-          docfiles << htmlfile
-        end
+  next unless $build_sdks.include?(RSDK)
+  tgt = Dir.pwd
+  Dir.mkdir("sdk/R")
+  Dir.mkdir("sdk/R/arvados")
+  puts("tgt", tgt)
+  cp('css/R.css', 'sdk/R/arvados')
+  docfiles = []
+  Dir.chdir("../sdk/R/") do
+    Dir.entries("man").each do |rd|
+      if rd[-3..-1] == ".Rd"
+        htmlfile = "#{rd[0..-4]}.html"
+        `R CMD Rdconv -t html man/#{rd} > #{tgt}/sdk/R/arvados/#{htmlfile}`
+        docfiles << htmlfile
       end
     end
-    raise if $? != 0
+  end
+  raise if $? != 0
 
-    File.open("sdk/R/README.md", "r") do |rd|
+  File.open("../sdk/R/README.md", "r") do |rd|
     File.open("sdk/R/index.html.md", "w") do |fn|
       fn.write(<<-EOF
 ---
@@ -66,14 +166,14 @@ navmenu: R
 title: "R SDK Overview"
 ...
 
-#{rd.read.gsub(/^```$/, "~~~").gsub(/^```(\w)$/, "~~~\\1")}
+#{rd.read}
 EOF
               )
-      end
     end
+  end
 
-    File.open("sdk/R/arvados/index.html.textile.liquid", "w") do |fn|
-      fn.write(<<-EOF
+  File.open("sdk/R/arvados/index.html.textile.liquid", "w") do |fn|
+    fn.write(<<-EOF
 ---
 layout: default
 navsection: sdk
@@ -87,55 +187,84 @@ SPDX-License-Identifier: CC-BY-SA-3.0
 {% endcomment %}
 
 EOF
-              )
-
-      docfiles.sort.each do |d|
-        fn.write("* \"#{d[0..-6]}\":#{d}\n")
-      end
-
+            )
+    docfiles.sort.each do |d|
+      fn.write("* \"#{d[0..-6]}\":#{d}\n")
     end
-  else
-    puts "Warning: R not found, R documentation will not be generated".colorize(:light_red)
   end
 end
 
 file "sdk/java-v2/javadoc/index.html" do |t|
-  if File.exists? "no-sdk"
-    next
-  end
-  `which java`
-  if $? == 0
-    `which gradle`
-    if $? != 0
-      puts "Warning: gradle not found, java sdk documentation will not be generated".colorize(:light_red)
-    else
-      tgt = Dir.pwd
-      docfiles = []
-      Dir.chdir("../sdk/java-v2") do
-        STDERR.puts `gradle javadoc 2>&1`
-        raise if $? != 0
-      end
-      cp_r("../sdk/java-v2/build/docs/javadoc", "sdk/java-v2")
-      raise if $? != 0
-    end
-  else
-    puts "Warning: java not found, java sdk documentation will not be generated".colorize(:light_red)
+  next unless $build_sdks.include?(JavaSDK)
+  tgt = Dir.pwd
+  docfiles = []
+  Dir.chdir("../sdk/java-v2") do
+    STDERR.puts `gradle javadoc 2>&1`
+    raise if $? != 0
+    puts `sed -i "s/@import.*dejavu.css.*//g" build/docs/javadoc/stylesheet.css`
+    raise if $? != 0
   end
+  cp_r("../sdk/java-v2/build/docs/javadoc", "sdk/java-v2")
+  raise if $? != 0
 end
 
 task :linkchecker => [ :generate ] do
-  Dir.chdir(".site") do
-    `which linkchecker`
-    if $? == 0
-      system "linkchecker index.html --ignore-url='!file://'" or exit $?.exitstatus
-    else
-      puts "Warning: linkchecker not found, skipping run".colorize(:light_red)
+  # we need --check-extern to check relative links, weird but true
+  opts = [
+    "--check-extern",
+    "--ignore-url=!^file://",
+  ]
+  ([JavaSDK, PythonSDK, RSDK] - $build_sdks).map(&:build_path).each do |sdk_path|
+    sdk_url = URI.join(ENV["baseurl"], sdk_path)
+    url_re = Regexp.escape(sdk_url.to_s)
+    opts << "--ignore-url=^#{url_re}[./]"
+  end
+  result = system(
+    "linkchecker", *opts, "index.html",
+    chdir: ".site",
+  )
+  if result.nil?
+    fail "could not run linkchecker command (is it installed?)"
+  elsif !result
+    fail "linkchecker exited #{$?.exitstatus}"
+  end
+end
+
+task :import_vscode_training do
+  Dir.chdir("user") do
+  rm_rf "arvados-vscode-cwl-training"
+  `git clone https://github.com/arvados/arvados-vscode-cwl-training`
+  githash = `git --git-dir arvados-vscode-cwl-training/.git log -n1 --format=%H HEAD`
+  File.open("cwl/arvados-vscode-training.html.md.liquid", "w") do |fn|
+    File.open("arvados-vscode-cwl-training/README.md", "r") do |rd|
+      fn.write(<<-EOF
+---
+layout: default
+navsection: userguide
+title: "Developing CWL Workflows with VSCode"
+...
+{% comment %}
+Copyright (C) The Arvados Authors. All rights reserved.
+
+SPDX-License-Identifier: CC-BY-SA-3.0
+
+Imported from https://github.com/arvados/arvados-vscode-cwl-training
+git hash: #{githash}
+{% endcomment %}
+
+EOF
+              )
+               fn.write(rd.read())
     end
   end
+  rm_rf "arvados-vscode-cwl-training"
+  end
 end
 
 task :clean do
   rm_rf "sdk/python/arvados"
+  rm_f "sdk/python/arvados.html"
+  rm_f "sdk/python/index.html"
   rm_rf "sdk/R"
   rm_rf "sdk/java-v2/javadoc"
 end
@@ -143,7 +272,8 @@ end
 require "zenweb/tasks"
 load "zenweb-textile.rb"
 load "zenweb-liquid.rb"
+load "zenweb-fix-body.rb"
 
 task :extra_wirings do
-  $website.pages["sdk/python/python.html.textile.liquid"].depends_on("sdk/python/arvados/index.html")
+  $website.pages["sdk/python/python.html.textile.liquid"].depends_on("sdk/python/arvados.html")
 end