arvados-version-server/arvados-version-server.go

   1 // Copyright (C) The Arvados Authors. All rights reserved.
   2 //
   3 // SPDX-License-Identifier: AGPL-3.0
   4
   5 package main
   6
   7 import (
   8         "encoding/json"
   9         "flag"
  10         "fmt"
  11         "git.curoverse.com/arvados.git/sdk/go/config"
  12         "io"
  13         "io/ioutil"
  14         "log"
  15         "net"
  16         "net/http"
  17         "os"
  18         "os/exec"
  19         "os/signal"
  20         "regexp"
  21         "strings"
  22         "syscall"
  23         "time"
  24 )
  25
  26 var listener net.Listener
  27
  28 type report struct {
  29         Type string
  30         Msg  string
  31 }
  32
  33 type bundle struct {
  34         sourceDir     string
  35         name          string
  36         packageType   string
  37         versionType   string
  38         versionPrefix string
  39 }
  40
  41 type result struct {
  42         RequestHash string
  43         GitHash     string
  44         Versions    map[string]map[string]string
  45         Cached      bool
  46         Elapsed     string
  47 }
  48
  49 type about struct {
  50         Name    string
  51         Version string
  52         URL     string
  53 }
  54
  55 type help struct {
  56         Usage string
  57 }
  58
  59 // Config structure
  60 type Config struct {
  61         DirPath           string
  62         CacheDirPath      string
  63         GitExecutablePath string
  64         ListenPort        string
  65
  66         Packages []bundle
  67 }
  68
  69 var theConfig Config
  70
  71 const defaultConfigPath = "/etc/arvados/version-server/version-server.yml"
  72
  73 func loadPackages() (packages []bundle) {
  74         packages = []bundle{
  75                 {
  76                         sourceDir:     ".",
  77                         name:          "arvados-src",
  78                         packageType:   "distribution",
  79                         versionType:   "git",
  80                         versionPrefix: "0.1",
  81                 },
  82                 {
  83                         sourceDir:     "apps/workbench",
  84                         name:          "arvados-workbench",
  85                         packageType:   "distribution",
  86                         versionType:   "git",
  87                         versionPrefix: "0.1",
  88                 },
  89                 {
  90                         sourceDir:     "sdk/cwl",
  91                         name:          "python-arvados-cwl-runner",
  92                         packageType:   "distribution",
  93                         versionType:   "python",
  94                         versionPrefix: "1.0",
  95                 },
  96                 {
  97                         sourceDir:     "sdk/cwl",
  98                         name:          "arvados-cwl-runner",
  99                         packageType:   "python",
 100                         versionType:   "python",
 101                         versionPrefix: "1.0",
 102                 },
 103                 {
 104                         sourceDir:     "sdk/cwl",
 105                         name:          "arvados/jobs",
 106                         packageType:   "docker",
 107                         versionType:   "docker",
 108                         versionPrefix: "",
 109                 },
 110                 {
 111                         sourceDir:     "sdk/go/crunchrunner",
 112                         name:          "crunchrunner",
 113                         packageType:   "distribution",
 114                         versionType:   "go",
 115                         versionPrefix: "0.1",
 116                 },
 117                 {
 118                         sourceDir:     "sdk/pam",
 119                         name:          "libpam-arvados",
 120                         packageType:   "distribution",
 121                         versionType:   "python",
 122                         versionPrefix: "0.1",
 123                 },
 124                 {
 125                         sourceDir:     "sdk/pam",
 126                         name:          "arvados-pam",
 127                         packageType:   "python",
 128                         versionType:   "python",
 129                         versionPrefix: "0.1",
 130                 },
 131                 {
 132                         sourceDir:     "sdk/python",
 133                         name:          "python-arvados-python-client",
 134                         packageType:   "distribution",
 135                         versionType:   "python",
 136                         versionPrefix: "0.1",
 137                 },
 138                 {
 139                         sourceDir:     "sdk/python",
 140                         name:          "arvados-python-client",
 141                         packageType:   "python",
 142                         versionType:   "python",
 143                         versionPrefix: "0.1",
 144                 },
 145                 {
 146                         sourceDir:     "services/api",
 147                         name:          "arvados-api-server",
 148                         packageType:   "distribution",
 149                         versionType:   "git",
 150                         versionPrefix: "0.1",
 151                 },
 152                 {
 153                         sourceDir:     "services/arv-git-httpd",
 154                         name:          "arvados-git-httpd",
 155                         packageType:   "distribution",
 156                         versionType:   "go",
 157                         versionPrefix: "0.1",
 158                 },
 159                 {
 160                         sourceDir:     "services/crunch-dispatch-local",
 161                         name:          "crunch-dispatch-local",
 162                         packageType:   "distribution",
 163                         versionType:   "go",
 164                         versionPrefix: "0.1",
 165                 },
 166                 {
 167                         sourceDir:     "services/crunch-dispatch-slurm",
 168                         name:          "crunch-dispatch-slurm",
 169                         packageType:   "distribution",
 170                         versionType:   "go",
 171                         versionPrefix: "0.1",
 172                 },
 173                 {
 174                         sourceDir:     "services/crunch-run",
 175                         name:          "crunch-run",
 176                         packageType:   "distribution",
 177                         versionType:   "go",
 178                         versionPrefix: "0.1",
 179                 },
 180                 {
 181                         sourceDir:     "services/crunchstat",
 182                         name:          "crunchstat",
 183                         packageType:   "distribution",
 184                         versionType:   "git",
 185                         versionPrefix: "0.1",
 186                 },
 187                 {
 188                         sourceDir:     "services/dockercleaner",
 189                         name:          "arvados-docker-cleaner",
 190                         packageType:   "distribution",
 191                         versionType:   "python",
 192                         versionPrefix: "0.1",
 193                 },
 194                 {
 195                         sourceDir:     "services/fuse",
 196                         name:          "python-arvados-fuse",
 197                         packageType:   "distribution",
 198                         versionType:   "python",
 199                         versionPrefix: "0.1",
 200                 },
 201                 {
 202                         sourceDir:     "services/fuse",
 203                         name:          "arvados_fuse",
 204                         packageType:   "python",
 205                         versionType:   "python",
 206                         versionPrefix: "0.1",
 207                 },
 208                 {
 209                         sourceDir:     "services/keep-balance",
 210                         name:          "keep-balance",
 211                         packageType:   "distribution",
 212                         versionType:   "go",
 213                         versionPrefix: "0.1",
 214                 },
 215                 {
 216                         sourceDir:     "services/keepproxy",
 217                         name:          "keepproxy",
 218                         packageType:   "distribution",
 219                         versionType:   "go",
 220                         versionPrefix: "0.1",
 221                 },
 222                 {
 223                         sourceDir:     "services/keepstore",
 224                         name:          "keepstore",
 225                         packageType:   "distribution",
 226                         versionType:   "go",
 227                         versionPrefix: "0.1",
 228                 },
 229                 {
 230                         sourceDir:     "services/keep-web",
 231                         name:          "keep-web",
 232                         packageType:   "distribution",
 233                         versionType:   "go",
 234                         versionPrefix: "0.1",
 235                 },
 236                 {
 237                         sourceDir:     "services/nodemanager",
 238                         name:          "arvados-node-manager",
 239                         packageType:   "distribution",
 240                         versionType:   "python",
 241                         versionPrefix: "0.1",
 242                 },
 243                 {
 244                         sourceDir:     "services/nodemanager",
 245                         name:          "arvados-node-manager",
 246                         packageType:   "python",
 247                         versionType:   "python",
 248                         versionPrefix: "0.1",
 249                 },
 250                 {
 251                         sourceDir:     "services/ws",
 252                         name:          "arvados-ws",
 253                         packageType:   "distribution",
 254                         versionType:   "go",
 255                         versionPrefix: "0.1",
 256                 },
 257                 {
 258                         sourceDir:     "tools/crunchstat-summary",
 259                         name:          "crunchstat-summary",
 260                         packageType:   "distribution",
 261                         versionType:   "go",
 262                         versionPrefix: "0.1",
 263                 },
 264                 {
 265                         sourceDir:     "tools/keep-block-check",
 266                         name:          "keep-block-check",
 267                         packageType:   "distribution",
 268                         versionType:   "go",
 269                         versionPrefix: "0.1",
 270                 },
 271                 {
 272                         sourceDir:     "tools/keep-exercise",
 273                         name:          "keep-exercise",
 274                         packageType:   "distribution",
 275                         versionType:   "go",
 276                         versionPrefix: "0.1",
 277                 },
 278                 {
 279                         sourceDir:     "tools/keep-rsync",
 280                         name:          "keep-rsync",
 281                         packageType:   "distribution",
 282                         versionType:   "go",
 283                         versionPrefix: "0.1",
 284                 },
 285                 {
 286                         sourceDir:     "sdk/ruby",
 287                         name:          "arvados",
 288                         packageType:   "gem",
 289                         versionType:   "ruby",
 290                         versionPrefix: "0.1",
 291                 },
 292                 {
 293                         sourceDir:     "sdk/cli",
 294                         name:          "arvados-cli",
 295                         packageType:   "gem",
 296                         versionType:   "ruby",
 297                         versionPrefix: "0.1",
 298                 },
 299                 {
 300                         sourceDir:     "services/login-sync",
 301                         name:          "arvados-login-sync",
 302                         packageType:   "gem",
 303                         versionType:   "ruby",
 304                         versionPrefix: "0.1",
 305                 },
 306         }
 307         return
 308 }
 309
 310 func lookupInCache(hash string) (result, error) {
 311         statData, err := os.Stat(theConfig.CacheDirPath)
 312         if os.IsNotExist(err) {
 313                 err = os.MkdirAll(theConfig.CacheDirPath, 0700)
 314                 if err != nil {
 315                         logError([]string{"Error creating directory", theConfig.CacheDirPath, ":", err.Error()})
 316                 }
 317         } else {
 318                 if !statData.IsDir() {
 319                         logError([]string{"The path", theConfig.CacheDirPath, "is not a directory"})
 320                         return result{}, fmt.Errorf("The path %s is not a directory", theConfig.CacheDirPath)
 321                 }
 322         }
 323         file, e := ioutil.ReadFile(theConfig.CacheDirPath + "/" + hash)
 324         if e != nil {
 325                 return result{}, fmt.Errorf("File error: %v\n", e)
 326         }
 327         var m result
 328         err = json.Unmarshal(file, &m)
 329         return m, err
 330 }
 331
 332 func writeToCache(hash string, data result) (err error) {
 333         statData, err := os.Stat(theConfig.CacheDirPath)
 334         if os.IsNotExist(err) {
 335                 err = os.MkdirAll(theConfig.CacheDirPath, 0700)
 336                 if err != nil {
 337                         logError([]string{"Error creating directory", theConfig.CacheDirPath, ":", err.Error()})
 338                 }
 339         } else {
 340                 if !statData.IsDir() {
 341                         logError([]string{"The path", theConfig.CacheDirPath, "is not a directory"})
 342                         return fmt.Errorf("The path %s is not a directory", theConfig.CacheDirPath)
 343                 }
 344         }
 345
 346         jsonData, err := json.Marshal(data)
 347         if err != nil {
 348                 return
 349         }
 350         err = ioutil.WriteFile(theConfig.CacheDirPath+"/"+hash, jsonData, 0644)
 351         return
 352 }
 353
 354 func prepareGitPath(hash string) error {
 355         statData, err := os.Stat(theConfig.DirPath)
 356         if os.IsNotExist(err) {
 357                 err = os.MkdirAll(theConfig.DirPath, 0700)
 358                 if err != nil {
 359                         logError([]string{"Error creating directory", theConfig.DirPath, ":", err.Error()})
 360                         return fmt.Errorf("Error creating directory %s", theConfig.DirPath)
 361                 }
 362                 cmdArgs := []string{"clone", "https://github.com/curoverse/arvados.git", theConfig.DirPath}
 363                 if _, err = exec.Command(theConfig.GitExecutablePath, cmdArgs...).Output(); err != nil {
 364                         logError([]string{"There was an error running the command ", theConfig.GitExecutablePath, strings.Join(cmdArgs, " "), err.Error()})
 365                         return fmt.Errorf("There was an error cloning the repository")
 366                 }
 367         } else {
 368                 if !statData.IsDir() {
 369                         logError([]string{"The path", theConfig.DirPath, "is not a directory"})
 370                         return fmt.Errorf("The path %s is not a directory", theConfig.DirPath)
 371                 }
 372         }
 373         return nil
 374 }
 375
 376 func prepareGitCheckout(hash string) (string, error) {
 377         err := prepareGitPath(hash)
 378         if err != nil {
 379                 return "", err
 380         }
 381         err = os.Chdir(theConfig.DirPath)
 382         if err != nil {
 383                 logError([]string{"Error changing directory to", theConfig.DirPath})
 384                 return "", fmt.Errorf("Error changing directory to %s", theConfig.DirPath)
 385         }
 386         cmdArgs := []string{"fetch", "--all"}
 387         if _, err := exec.Command(theConfig.GitExecutablePath, cmdArgs...).Output(); err != nil {
 388                 logError([]string{"There was an error running the command ", theConfig.GitExecutablePath, strings.Join(cmdArgs, " "), err.Error()})
 389                 return "", fmt.Errorf("There was an error fetching all remotes")
 390         }
 391         if hash == "" {
 392                 hash = "master"
 393         }
 394         cmdArgs = []string{"checkout", hash}
 395         if _, err := exec.Command(theConfig.GitExecutablePath, cmdArgs...).Output(); err != nil {
 396                 logError([]string{"There was an error running the command ", theConfig.GitExecutablePath, strings.Join(cmdArgs, " "), err.Error()})
 397                 return "", fmt.Errorf("There was an error checking out the requested revision")
 398         }
 399         if hash == "master" {
 400                 cmdArgs := []string{"reset", "--hard", "origin/master"}
 401                 if _, err := exec.Command(theConfig.GitExecutablePath, cmdArgs...).Output(); err != nil {
 402                         logError([]string{"There was an error running the command ", theConfig.GitExecutablePath, strings.Join(cmdArgs, " "), err.Error()})
 403                         return "", fmt.Errorf("There was an error fetching all remotes")
 404                 }
 405         }
 406         return "", nil
 407 }
 408
 409 // Generates the hash for the latest git commit for the current working directory
 410 func gitHashFull() (string, error) {
 411         cmdArgs := []string{"log", "-n1", "--first-parent", "--max-count=1", "--format=format:%H", "."}
 412         cmdOut, err := exec.Command(theConfig.GitExecutablePath, cmdArgs...).Output()
 413         if err != nil {
 414                 logError([]string{"There was an error running the command ", theConfig.GitExecutablePath, strings.Join(cmdArgs, " "), err.Error()})
 415                 return "", fmt.Errorf("There was an error getting the git hash for this revision")
 416         }
 417         return string(cmdOut), nil
 418 }
 419
 420 // Generates a version number from the git log for the current working directory
 421 func versionFromGit(prefix string) (string, error) {
 422         gitTs, err := getGitTs()
 423         if err != nil {
 424                 return "", err
 425         }
 426         cmdArgs := []string{"log", "-n1", "--first-parent", "--max-count=1", "--format=format:%h", "."}
 427         gitHash, err := exec.Command(theConfig.GitExecutablePath, cmdArgs...).Output()
 428         if err != nil {
 429                 logError([]string{"There was an error running the command ", theConfig.GitExecutablePath, strings.Join(cmdArgs, " "), err.Error()})
 430                 return "", fmt.Errorf("There was an error getting the git hash for this revision")
 431         }
 432         cmdName := "/bin/date"
 433         cmdArgs = []string{"-ud", "@" + string(gitTs), "+%Y%m%d%H%M%S"}
 434         date, err := exec.Command(cmdName, cmdArgs...).Output()
 435         if err != nil {
 436                 logError([]string{"There was an error running the command ", cmdName, strings.Join(cmdArgs, " "), err.Error()})
 437                 return "", fmt.Errorf("There was an error converting the datestamp for this revision")
 438         }
 439
 440         return fmt.Sprintf("%s.%s.%s", strings.TrimSpace(prefix), strings.TrimSpace(string(date)), strings.TrimSpace(string(gitHash))), nil
 441 }
 442
 443 // Generates a python package version number from the git log for the current working directory
 444 func rubyVersionFromGit(prefix string) (string, error) {
 445         gitTs, err := getGitTs()
 446         if err != nil {
 447                 return "", err
 448         }
 449         cmdName := "/bin/date"
 450         cmdArgs := []string{"-ud", "@" + string(gitTs), "+%Y%m%d%H%M%S"}
 451         date, err := exec.Command(cmdName, cmdArgs...).Output()
 452         if err != nil {
 453                 logError([]string{"There was an error running the command ", cmdName, strings.Join(cmdArgs, " "), err.Error()})
 454                 return "", fmt.Errorf("There was an error converting the datestamp for this revision")
 455         }
 456
 457         return fmt.Sprintf("%s.%s", strings.TrimSpace(prefix), strings.TrimSpace(string(date))), nil
 458 }
 459
 460 // Generates a python package version number from the git log for the current working directory
 461 func pythonVersionFromGit(prefix string) (string, error) {
 462         rv, err := rubyVersionFromGit(prefix)
 463         if err != nil {
 464                 return "", err
 465         }
 466         return rv, nil
 467 }
 468
 469 // Generates a docker image version number from the git log for the current working directory
 470 func dockerVersionFromGit() (string, error) {
 471         rv, err := gitHashFull()
 472         if err != nil {
 473                 return "", err
 474         }
 475         return rv, nil
 476 }
 477
 478 func getGitTs() (gitTs []byte, err error) {
 479         cmdArgs := []string{"log", "-n1", "--first-parent", "--max-count=1", "--format=format:%ct", "."}
 480         gitTs, err = exec.Command(theConfig.GitExecutablePath, cmdArgs...).Output()
 481         if err != nil {
 482                 logError([]string{"There was an error running the command ", theConfig.GitExecutablePath, strings.Join(cmdArgs, " "), err.Error()})
 483                 return nil, fmt.Errorf("There was an error getting the git hash for this revision")
 484         }
 485         return
 486 }
 487
 488 // Generates a timestamp from the git log for the current working directory
 489 func timestampFromGit() (string, error) {
 490         gitTs, err := getGitTs()
 491         if err != nil {
 492                 return "", err
 493         }
 494         return fmt.Sprintf("%s", strings.TrimSpace(string(gitTs))), nil
 495 }
 496
 497 func normalizeRequestedHash(hash string) (string, error) {
 498         _, err := prepareGitCheckout(hash)
 499         if err != nil {
 500                 return "", err
 501         }
 502
 503         // Get the git hash for the tree
 504         var gitHash string
 505         gitHash, err = gitHashFull()
 506         if err != nil {
 507                 return "", err
 508         }
 509
 510         return gitHash, nil
 511 }
 512
 513 func getPackageVersionsWorker(hash string) (gitHash string, goSDKTimestamp string, goSDKVersionWithoutPrefix string, pythonSDKTimestamp string, err error) {
 514         _, err = prepareGitCheckout(hash)
 515         if err != nil {
 516                 return "", "", "", "", err
 517         }
 518
 519         // Get the git hash for the tree
 520         gitHash, err = gitHashFull()
 521         if err != nil {
 522                 return "", "", "", "", err
 523         }
 524
 525         // Get the git timestamp and version string for the sdk/go directory
 526         err = os.Chdir(theConfig.DirPath + "/sdk/go")
 527         if err != nil {
 528                 goSDKTimestamp = ""
 529                 goSDKVersionWithoutPrefix = ""
 530                 err = nil
 531         } else {
 532                 goSDKTimestamp, err = timestampFromGit()
 533                 if err != nil {
 534                         return "", "", "", "", err
 535                 }
 536                 goSDKVersionWithoutPrefix, err = versionFromGit("")
 537                 if err != nil {
 538                         return "", "", "", "", err
 539                 }
 540         }
 541
 542         // Get the git timestamp and version string for the sdk/python directory
 543         err = os.Chdir(theConfig.DirPath + "/sdk/python")
 544         if err != nil {
 545                 pythonSDKTimestamp = ""
 546                 err = nil
 547         } else {
 548                 pythonSDKTimestamp, err = timestampFromGit()
 549                 if err != nil {
 550                         return "", "", "", "", err
 551                 }
 552         }
 553
 554         return
 555 }
 556
 557 func pythonSDKVersionCheck(pythonSDKTimestamp string) (err error) {
 558         var packageTimestamp string
 559         packageTimestamp, err = timestampFromGit()
 560         if err != nil {
 561                 return
 562         }
 563
 564         if pythonSDKTimestamp > packageTimestamp {
 565                 err = os.Chdir(theConfig.DirPath + "/sdk/python")
 566                 if err != nil {
 567                         return
 568                 }
 569         }
 570         return
 571 }
 572
 573 func getPackageVersions(hash string) (versions map[string]map[string]string, gitHash string, err error) {
 574         versions = make(map[string]map[string]string)
 575
 576         gitHash, goSDKTimestamp, goSDKVersionWithoutPrefix, pythonSDKTimestamp, err := getPackageVersionsWorker(hash)
 577         if err != nil {
 578                 return nil, "", err
 579         }
 580
 581         for _, p := range theConfig.Packages {
 582                 err = os.Chdir(theConfig.DirPath + "/" + p.sourceDir)
 583                 if err != nil {
 584                         // Skip those packages for which the source directory doesn't exist
 585                         // in this revision of the source tree.
 586                         err = nil
 587                         continue
 588                 }
 589                 name := p.name
 590
 591                 var packageVersion string
 592
 593                 if (p.versionType == "git") || (p.versionType == "go") {
 594                         packageVersion, err = versionFromGit(p.versionPrefix)
 595                         if err != nil {
 596                                 return nil, "", err
 597                         }
 598                 }
 599                 if p.versionType == "go" {
 600                         var packageTimestamp string
 601                         packageTimestamp, err = timestampFromGit()
 602                         if err != nil {
 603                                 return nil, "", err
 604                         }
 605
 606                         if goSDKTimestamp > packageTimestamp {
 607                                 packageVersion = p.versionPrefix + goSDKVersionWithoutPrefix
 608                         }
 609                 } else if p.versionType == "python" {
 610                         // Not all of our packages that use our python sdk are automatically
 611                         // getting rebuilt when sdk/python changes. Yet.
 612                         if p.name == "python-arvados-cwl-runner" {
 613                                 err = pythonSDKVersionCheck(pythonSDKTimestamp)
 614                                 if err != nil {
 615                                         return nil, "", err
 616                                 }
 617                         }
 618
 619                         packageVersion, err = pythonVersionFromGit(p.versionPrefix)
 620                         if err != nil {
 621                                 return nil, "", err
 622                         }
 623                 } else if p.versionType == "ruby" {
 624                         packageVersion, err = rubyVersionFromGit(p.versionPrefix)
 625                         if err != nil {
 626                                 return nil, "", err
 627                         }
 628                 } else if p.versionType == "docker" {
 629                         // the arvados/jobs image version is always the latest of the
 630                         // sdk/python and the sdk/cwl version
 631                         if p.name == "arvados/jobs" {
 632                                 err = pythonSDKVersionCheck(pythonSDKTimestamp)
 633                                 if err != nil {
 634                                         return nil, "", err
 635                                 }
 636                         }
 637                         packageVersion, err = dockerVersionFromGit()
 638                         if err != nil {
 639                                 return nil, "", err
 640                         }
 641                 }
 642
 643                 if versions[strings.Title(p.packageType)] == nil {
 644                         versions[strings.Title(p.packageType)] = make(map[string]string)
 645                 }
 646                 versions[strings.Title(p.packageType)][name] = packageVersion
 647         }
 648
 649         return
 650 }
 651
 652 func logError(m []string) {
 653         log.Printf(string(marshal(report{"Error", strings.Join(m, " ")})))
 654 }
 655
 656 func logNotice(m []string) {
 657         log.Printf(string(marshal(report{"Notice", strings.Join(m, " ")})))
 658 }
 659
 660 func marshal(message interface{}) (encoded []byte) {
 661         encoded, err := json.Marshal(message)
 662         if err != nil {
 663                 // do not call logError here because that would create an infinite loop
 664                 fmt.Fprintln(os.Stderr, "{\"Error\": \"Unable to marshal message into json:", message, "\"}")
 665                 return nil
 666         }
 667         return
 668 }
 669
 670 func marshalAndWrite(w io.Writer, message interface{}) {
 671         b := marshal(message)
 672         if b == nil {
 673                 errorMessage := "{\n\"Error\": \"Unspecified error\"\n}"
 674                 _, err := io.WriteString(w, errorMessage)
 675                 if err != nil {
 676                         // do not call logError (it calls marshal and that function has already failed at this point)
 677                         fmt.Fprintln(os.Stderr, "{\"Error\": \"Unable to write message to client\"}")
 678                 }
 679         } else {
 680                 _, err := w.Write(b)
 681                 if err != nil {
 682                         logError([]string{"Unable to write message to client:", string(b)})
 683                 }
 684         }
 685 }
 686
 687 func packageVersionHandler(w http.ResponseWriter, r *http.Request) {
 688         start := time.Now()
 689         w.Header().Set("Content-Type", "application/json; charset=utf-8")
 690
 691         var packageVersions map[string]map[string]string
 692         var cached bool
 693
 694         // Sanity check the input RequestHash
 695         match, err := regexp.MatchString("^([a-z0-9]+|)$", r.URL.Path[11:])
 696         if err != nil {
 697                 m := report{"Error", "Error matching RequestHash"}
 698                 marshalAndWrite(w, m)
 699                 return
 700         }
 701         if !match {
 702                 m := report{"Error", "Invalid RequestHash"}
 703                 marshalAndWrite(w, m)
 704                 return
 705         }
 706
 707         hash := r.URL.Path[11:]
 708
 709         // Empty hash or non-standard hash length? Normalize it.
 710         if len(hash) != 7 && len(hash) != 40 {
 711                 hash, err = normalizeRequestedHash(hash)
 712                 if err != nil {
 713                         m := report{"Error", err.Error()}
 714                         marshalAndWrite(w, m)
 715                         return
 716                 }
 717         }
 718
 719         var gitHash string
 720         rs, err := lookupInCache(hash)
 721         if err == nil {
 722                 packageVersions = rs.Versions
 723                 gitHash = rs.GitHash
 724                 cached = true
 725         } else {
 726                 packageVersions, gitHash, err = getPackageVersions(hash)
 727                 if err != nil {
 728                         m := report{"Error", err.Error()}
 729                         marshalAndWrite(w, m)
 730                         return
 731                 }
 732                 m := result{"", gitHash, packageVersions, true, ""}
 733                 err = writeToCache(hash, m)
 734                 if err != nil {
 735                         logError([]string{"Unable to save entry in cache directory", theConfig.CacheDirPath})
 736                 }
 737                 cached = false
 738         }
 739
 740         m := result{hash, gitHash, packageVersions, cached, fmt.Sprintf("%v", time.Since(start))}
 741         marshalAndWrite(w, m)
 742 }
 743
 744 func aboutHandler(w http.ResponseWriter, r *http.Request) {
 745         w.Header().Set("Content-Type", "application/json; charset=utf-8")
 746         m := about{"Arvados Version Server", "0.1", "https://arvados.org"}
 747         marshalAndWrite(w, m)
 748 }
 749
 750 func helpHandler(w http.ResponseWriter, r *http.Request) {
 751         w.Header().Set("Content-Type", "application/json; charset=utf-8")
 752         m := help{"GET /v1/commit/ or GET /v1/commit/git-commit or GET /v1/about or GET /v1/help"}
 753         marshalAndWrite(w, m)
 754 }
 755
 756 func parseFlags() (configPath *string) {
 757
 758         flags := flag.NewFlagSet("arvados-version-server", flag.ExitOnError)
 759         flags.Usage = func() { usage(flags) }
 760
 761         configPath = flags.String(
 762                 "config",
 763                 defaultConfigPath,
 764                 "`path` to YAML configuration file")
 765
 766         // Parse args; omit the first arg which is the command name
 767         err := flags.Parse(os.Args[1:])
 768         if err != nil {
 769                 logError([]string{"Unable to parse command line arguments:", err.Error()})
 770                 os.Exit(1)
 771         }
 772
 773         return
 774 }
 775
 776 func main() {
 777         err := os.Setenv("TZ", "UTC")
 778         if err != nil {
 779                 logError([]string{"Error setting environment variable:", err.Error()})
 780                 os.Exit(1)
 781         }
 782
 783         configPath := parseFlags()
 784
 785         err = readConfig(&theConfig, *configPath, defaultConfigPath)
 786         if err != nil {
 787                 logError([]string{"Unable to start Arvados Version Server:", err.Error()})
 788                 os.Exit(1)
 789         }
 790
 791         theConfig.Packages = loadPackages()
 792
 793         if theConfig.DirPath == "" {
 794                 theConfig.DirPath = "/tmp/arvados-version-server-checkout"
 795         }
 796
 797         if theConfig.CacheDirPath == "" {
 798                 theConfig.CacheDirPath = "/tmp/arvados-version-server-cache"
 799         }
 800
 801         if theConfig.GitExecutablePath == "" {
 802                 theConfig.GitExecutablePath = "/usr/bin/git"
 803         }
 804
 805         if theConfig.ListenPort == "" {
 806                 theConfig.ListenPort = "80"
 807         }
 808
 809         http.HandleFunc("/v1/commit/", packageVersionHandler)
 810         http.HandleFunc("/v1/about", aboutHandler)
 811         http.HandleFunc("/v1/help", helpHandler)
 812         http.HandleFunc("/v1", helpHandler)
 813         http.HandleFunc("/", helpHandler)
 814         logNotice([]string{"Arvados Version Server listening on port", theConfig.ListenPort})
 815
 816         listener, err = net.Listen("tcp", ":"+theConfig.ListenPort)
 817
 818         if err != nil {
 819                 logError([]string{"Unable to start Arvados Version Server:", err.Error()})
 820                 os.Exit(1)
 821         }
 822
 823         // Shut down the server gracefully (by closing the listener)
 824         // if SIGTERM is received.
 825         term := make(chan os.Signal, 1)
 826         go func(sig <-chan os.Signal) {
 827                 <-sig
 828                 logError([]string{"caught signal"})
 829                 _ = listener.Close()
 830         }(term)
 831         signal.Notify(term, syscall.SIGTERM)
 832         signal.Notify(term, syscall.SIGINT)
 833
 834         // Start serving requests.
 835         _ = http.Serve(listener, nil)
 836         // http.Serve returns an error when it gets the term or int signal
 837
 838         logNotice([]string{"Arvados Version Server shutting down"})
 839
 840 }
 841
 842 func readConfig(cfg interface{}, path string, defaultConfigPath string) error {
 843         err := config.LoadFile(cfg, path)
 844         if err != nil && os.IsNotExist(err) && path == defaultConfigPath {
 845                 logNotice([]string{"Config not specified. Continue with default configuration."})
 846                 err = nil
 847         }
 848         return err
 849 }