arvados-version-server/arvados-version-server.go

   1 // Copyright (C) The Arvados Authors. All rights reserved.
   2 //
   3 // SPDX-License-Identifier: AGPL-3.0
   4
   5 package main
   6
   7 import (
   8         "encoding/json"
   9         "flag"
  10         "fmt"
  11         "git.curoverse.com/arvados.git/sdk/go/config"
  12         "io"
  13         "io/ioutil"
  14         "net"
  15         "net/http"
  16         "os"
  17         "os/exec"
  18         "os/signal"
  19         "regexp"
  20         "strings"
  21         "syscall"
  22         "time"
  23 )
  24
  25 var listener net.Listener
  26
  27 type logStruct struct {
  28         Type string
  29         Msg  string
  30 }
  31
  32 type packageStruct struct {
  33         sourceDir            string
  34         packageName          string
  35         packageType          string
  36         packageVersionType   string
  37         packageVersionPrefix string
  38 }
  39
  40 type returnStruct struct {
  41         RequestHash string
  42         GitHash     string
  43         Versions    map[string]map[string]string
  44         Cached      bool
  45         Elapsed     string
  46 }
  47
  48 type aboutStruct struct {
  49         Name    string
  50         Version string
  51         URL     string
  52 }
  53
  54 type helpStruct struct {
  55         Usage string
  56 }
  57
  58 // Config structure
  59 type Config struct {
  60         DirPath           string
  61         CacheDirPath      string
  62         GitExecutablePath string
  63         ListenPort        string
  64
  65         Packages []packageStruct
  66 }
  67
  68 var theConfig Config
  69
  70 const defaultConfigPath = "/etc/arvados/version-server/version-server.yml"
  71
  72 func loadPackages() (packages []packageStruct) {
  73         packages = []packageStruct{
  74                 {
  75                         sourceDir:            ".",
  76                         packageName:          "arvados-src",
  77                         packageType:          "distribution",
  78                         packageVersionType:   "git",
  79                         packageVersionPrefix: "0.1",
  80                 },
  81                 {
  82                         sourceDir:            "apps/workbench",
  83                         packageName:          "arvados-workbench",
  84                         packageType:          "distribution",
  85                         packageVersionType:   "git",
  86                         packageVersionPrefix: "0.1",
  87                 },
  88                 {
  89                         sourceDir:            "sdk/cwl",
  90                         packageName:          "python-arvados-cwl-runner",
  91                         packageType:          "distribution",
  92                         packageVersionType:   "python",
  93                         packageVersionPrefix: "1.0",
  94                 },
  95                 {
  96                         sourceDir:            "sdk/cwl",
  97                         packageName:          "arvados-cwl-runner",
  98                         packageType:          "python",
  99                         packageVersionType:   "python",
 100                         packageVersionPrefix: "1.0",
 101                 },
 102                 {
 103                         sourceDir:            "sdk/cwl",
 104                         packageName:          "arvados/jobs",
 105                         packageType:          "docker",
 106                         packageVersionType:   "docker",
 107                         packageVersionPrefix: "",
 108                 },
 109                 {
 110                         sourceDir:            "sdk/go/crunchrunner",
 111                         packageName:          "crunchrunner",
 112                         packageType:          "distribution",
 113                         packageVersionType:   "go",
 114                         packageVersionPrefix: "0.1",
 115                 },
 116                 {
 117                         sourceDir:            "sdk/pam",
 118                         packageName:          "libpam-arvados",
 119                         packageType:          "distribution",
 120                         packageVersionType:   "python",
 121                         packageVersionPrefix: "0.1",
 122                 },
 123                 {
 124                         sourceDir:            "sdk/pam",
 125                         packageName:          "arvados-pam",
 126                         packageType:          "python",
 127                         packageVersionType:   "python",
 128                         packageVersionPrefix: "0.1",
 129                 },
 130                 {
 131                         sourceDir:            "sdk/python",
 132                         packageName:          "python-arvados-python-client",
 133                         packageType:          "distribution",
 134                         packageVersionType:   "python",
 135                         packageVersionPrefix: "0.1",
 136                 },
 137                 {
 138                         sourceDir:            "sdk/python",
 139                         packageName:          "arvados-python-client",
 140                         packageType:          "python",
 141                         packageVersionType:   "python",
 142                         packageVersionPrefix: "0.1",
 143                 },
 144                 {
 145                         sourceDir:            "services/api",
 146                         packageName:          "arvados-api-server",
 147                         packageType:          "distribution",
 148                         packageVersionType:   "git",
 149                         packageVersionPrefix: "0.1",
 150                 },
 151                 {
 152                         sourceDir:            "services/arv-git-httpd",
 153                         packageName:          "arvados-git-httpd",
 154                         packageType:          "distribution",
 155                         packageVersionType:   "go",
 156                         packageVersionPrefix: "0.1",
 157                 },
 158                 {
 159                         sourceDir:            "services/crunch-dispatch-local",
 160                         packageName:          "crunch-dispatch-local",
 161                         packageType:          "distribution",
 162                         packageVersionType:   "go",
 163                         packageVersionPrefix: "0.1",
 164                 },
 165                 {
 166                         sourceDir:            "services/crunch-dispatch-slurm",
 167                         packageName:          "crunch-dispatch-slurm",
 168                         packageType:          "distribution",
 169                         packageVersionType:   "go",
 170                         packageVersionPrefix: "0.1",
 171                 },
 172                 {
 173                         sourceDir:            "services/crunch-run",
 174                         packageName:          "crunch-run",
 175                         packageType:          "distribution",
 176                         packageVersionType:   "go",
 177                         packageVersionPrefix: "0.1",
 178                 },
 179                 {
 180                         sourceDir:            "services/crunchstat",
 181                         packageName:          "crunchstat",
 182                         packageType:          "distribution",
 183                         packageVersionType:   "git",
 184                         packageVersionPrefix: "0.1",
 185                 },
 186                 {
 187                         sourceDir:            "services/dockercleaner",
 188                         packageName:          "arvados-docker-cleaner",
 189                         packageType:          "distribution",
 190                         packageVersionType:   "python",
 191                         packageVersionPrefix: "0.1",
 192                 },
 193                 {
 194                         sourceDir:            "services/fuse",
 195                         packageName:          "python-arvados-fuse",
 196                         packageType:          "distribution",
 197                         packageVersionType:   "python",
 198                         packageVersionPrefix: "0.1",
 199                 },
 200                 {
 201                         sourceDir:            "services/fuse",
 202                         packageName:          "arvados_fuse",
 203                         packageType:          "python",
 204                         packageVersionType:   "python",
 205                         packageVersionPrefix: "0.1",
 206                 },
 207                 {
 208                         sourceDir:            "services/keep-balance",
 209                         packageName:          "keep-balance",
 210                         packageType:          "distribution",
 211                         packageVersionType:   "go",
 212                         packageVersionPrefix: "0.1",
 213                 },
 214                 {
 215                         sourceDir:            "services/keepproxy",
 216                         packageName:          "keepproxy",
 217                         packageType:          "distribution",
 218                         packageVersionType:   "go",
 219                         packageVersionPrefix: "0.1",
 220                 },
 221                 {
 222                         sourceDir:            "services/keepstore",
 223                         packageName:          "keepstore",
 224                         packageType:          "distribution",
 225                         packageVersionType:   "go",
 226                         packageVersionPrefix: "0.1",
 227                 },
 228                 {
 229                         sourceDir:            "services/keep-web",
 230                         packageName:          "keep-web",
 231                         packageType:          "distribution",
 232                         packageVersionType:   "go",
 233                         packageVersionPrefix: "0.1",
 234                 },
 235                 {
 236                         sourceDir:            "services/nodemanager",
 237                         packageName:          "arvados-node-manager",
 238                         packageType:          "distribution",
 239                         packageVersionType:   "python",
 240                         packageVersionPrefix: "0.1",
 241                 },
 242                 {
 243                         sourceDir:            "services/nodemanager",
 244                         packageName:          "arvados-node-manager",
 245                         packageType:          "python",
 246                         packageVersionType:   "python",
 247                         packageVersionPrefix: "0.1",
 248                 },
 249                 {
 250                         sourceDir:            "services/ws",
 251                         packageName:          "arvados-ws",
 252                         packageType:          "distribution",
 253                         packageVersionType:   "go",
 254                         packageVersionPrefix: "0.1",
 255                 },
 256                 {
 257                         sourceDir:            "tools/crunchstat-summary",
 258                         packageName:          "crunchstat-summary",
 259                         packageType:          "distribution",
 260                         packageVersionType:   "go",
 261                         packageVersionPrefix: "0.1",
 262                 },
 263                 {
 264                         sourceDir:            "tools/keep-block-check",
 265                         packageName:          "keep-block-check",
 266                         packageType:          "distribution",
 267                         packageVersionType:   "go",
 268                         packageVersionPrefix: "0.1",
 269                 },
 270                 {
 271                         sourceDir:            "tools/keep-exercise",
 272                         packageName:          "keep-exercise",
 273                         packageType:          "distribution",
 274                         packageVersionType:   "go",
 275                         packageVersionPrefix: "0.1",
 276                 },
 277                 {
 278                         sourceDir:            "tools/keep-rsync",
 279                         packageName:          "keep-rsync",
 280                         packageType:          "distribution",
 281                         packageVersionType:   "go",
 282                         packageVersionPrefix: "0.1",
 283                 },
 284                 {
 285                         sourceDir:            "sdk/ruby",
 286                         packageName:          "arvados",
 287                         packageType:          "gem",
 288                         packageVersionType:   "ruby",
 289                         packageVersionPrefix: "0.1",
 290                 },
 291                 {
 292                         sourceDir:            "sdk/cli",
 293                         packageName:          "arvados-cli",
 294                         packageType:          "gem",
 295                         packageVersionType:   "ruby",
 296                         packageVersionPrefix: "0.1",
 297                 },
 298                 {
 299                         sourceDir:            "services/login-sync",
 300                         packageName:          "arvados-login-sync",
 301                         packageType:          "gem",
 302                         packageVersionType:   "ruby",
 303                         packageVersionPrefix: "0.1",
 304                 },
 305         }
 306         return
 307 }
 308
 309 func lookupInCache(hash string) (returnStruct, error) {
 310         statData, err := os.Stat(theConfig.CacheDirPath)
 311         if os.IsNotExist(err) {
 312                 err = os.MkdirAll(theConfig.CacheDirPath, 0700)
 313                 if err != nil {
 314                         logError([]string{"Error creating directory", theConfig.CacheDirPath, ":", err.Error()})
 315                 }
 316         } else {
 317                 if !statData.IsDir() {
 318                         logError([]string{"The path", theConfig.CacheDirPath, "is not a directory"})
 319                         return returnStruct{}, fmt.Errorf("The path %s is not a directory", theConfig.CacheDirPath)
 320                 }
 321         }
 322         file, e := ioutil.ReadFile(theConfig.CacheDirPath + "/" + hash)
 323         if e != nil {
 324                 return returnStruct{}, fmt.Errorf("File error: %v\n", e)
 325         }
 326         var m returnStruct
 327         err = json.Unmarshal(file, &m)
 328         return m, err
 329 }
 330
 331 func writeToCache(hash string, data returnStruct) (err error) {
 332         statData, err := os.Stat(theConfig.CacheDirPath)
 333         if os.IsNotExist(err) {
 334                 err = os.MkdirAll(theConfig.CacheDirPath, 0700)
 335                 if err != nil {
 336                         logError([]string{"Error creating directory", theConfig.CacheDirPath, ":", err.Error()})
 337                 }
 338         } else {
 339                 if !statData.IsDir() {
 340                         logError([]string{"The path", theConfig.CacheDirPath, "is not a directory"})
 341                         return fmt.Errorf("The path %s is not a directory", theConfig.CacheDirPath)
 342                 }
 343         }
 344
 345         jsonData, err := json.Marshal(data)
 346         if err != nil {
 347                 return
 348         }
 349         err = ioutil.WriteFile(theConfig.CacheDirPath+"/"+hash, jsonData, 0644)
 350         return
 351 }
 352
 353 func prepareGitPath(hash string) error {
 354         statData, err := os.Stat(theConfig.DirPath)
 355         if os.IsNotExist(err) {
 356                 err = os.MkdirAll(theConfig.DirPath, 0700)
 357                 if err != nil {
 358                         logError([]string{"Error creating directory", theConfig.DirPath, ":", err.Error()})
 359                         return fmt.Errorf("Error creating directory %s", theConfig.DirPath)
 360                 }
 361                 cmdArgs := []string{"clone", "https://github.com/curoverse/arvados.git", theConfig.DirPath}
 362                 if _, err = exec.Command(theConfig.GitExecutablePath, cmdArgs...).Output(); err != nil {
 363                         logError([]string{"There was an error running the command ", theConfig.GitExecutablePath, strings.Join(cmdArgs, " "), err.Error()})
 364                         return fmt.Errorf("There was an error cloning the repository")
 365                 }
 366         } else {
 367                 if !statData.IsDir() {
 368                         logError([]string{"The path", theConfig.DirPath, "is not a directory"})
 369                         return fmt.Errorf("The path %s is not a directory", theConfig.DirPath)
 370                 }
 371         }
 372         return nil
 373 }
 374
 375 func prepareGitCheckout(hash string) (string, error) {
 376         err := prepareGitPath(hash)
 377         if err != nil {
 378                 return "", err
 379         }
 380         err = os.Chdir(theConfig.DirPath)
 381         if err != nil {
 382                 logError([]string{"Error changing directory to", theConfig.DirPath})
 383                 return "", fmt.Errorf("Error changing directory to %s", theConfig.DirPath)
 384         }
 385         cmdArgs := []string{"fetch", "--all"}
 386         if _, err := exec.Command(theConfig.GitExecutablePath, cmdArgs...).Output(); err != nil {
 387                 logError([]string{"There was an error running the command ", theConfig.GitExecutablePath, strings.Join(cmdArgs, " "), err.Error()})
 388                 return "", fmt.Errorf("There was an error fetching all remotes")
 389         }
 390         if hash == "" {
 391                 hash = "master"
 392         }
 393         cmdArgs = []string{"checkout", hash}
 394         if _, err := exec.Command(theConfig.GitExecutablePath, cmdArgs...).Output(); err != nil {
 395                 logError([]string{"There was an error running the command ", theConfig.GitExecutablePath, strings.Join(cmdArgs, " "), err.Error()})
 396                 return "", fmt.Errorf("There was an error checking out the requested revision")
 397         }
 398         if hash == "master" {
 399                 cmdArgs := []string{"reset", "--hard", "origin/master"}
 400                 if _, err := exec.Command(theConfig.GitExecutablePath, cmdArgs...).Output(); err != nil {
 401                         logError([]string{"There was an error running the command ", theConfig.GitExecutablePath, strings.Join(cmdArgs, " "), err.Error()})
 402                         return "", fmt.Errorf("There was an error fetching all remotes")
 403                 }
 404         }
 405         return "", nil
 406 }
 407
 408 // Generates the hash for the latest git commit for the current working directory
 409 func gitHashFull() (string, error) {
 410         cmdArgs := []string{"log", "-n1", "--first-parent", "--max-count=1", "--format=format:%H", "."}
 411         cmdOut, err := exec.Command(theConfig.GitExecutablePath, cmdArgs...).Output()
 412         if err != nil {
 413                 logError([]string{"There was an error running the command ", theConfig.GitExecutablePath, strings.Join(cmdArgs, " "), err.Error()})
 414                 return "", fmt.Errorf("There was an error getting the git hash for this revision")
 415         }
 416         return string(cmdOut), nil
 417 }
 418
 419 // Generates a version number from the git log for the current working directory
 420 func versionFromGit(prefix string) (string, error) {
 421         gitTs, err := getGitTs()
 422         if err != nil {
 423                 return "", err
 424         }
 425         cmdArgs := []string{"log", "-n1", "--first-parent", "--max-count=1", "--format=format:%h", "."}
 426         gitHash, err := exec.Command(theConfig.GitExecutablePath, cmdArgs...).Output()
 427         if err != nil {
 428                 logError([]string{"There was an error running the command ", theConfig.GitExecutablePath, strings.Join(cmdArgs, " "), err.Error()})
 429                 return "", fmt.Errorf("There was an error getting the git hash for this revision")
 430         }
 431         cmdName := "/bin/date"
 432         cmdArgs = []string{"-ud", "@" + string(gitTs), "+%Y%m%d%H%M%S"}
 433         date, err := exec.Command(cmdName, cmdArgs...).Output()
 434         if err != nil {
 435                 logError([]string{"There was an error running the command ", cmdName, strings.Join(cmdArgs, " "), err.Error()})
 436                 return "", fmt.Errorf("There was an error converting the datestamp for this revision")
 437         }
 438
 439         return fmt.Sprintf("%s.%s.%s", strings.TrimSpace(prefix), strings.TrimSpace(string(date)), strings.TrimSpace(string(gitHash))), nil
 440 }
 441
 442 // Generates a python package version number from the git log for the current working directory
 443 func rubyVersionFromGit(prefix string) (string, error) {
 444         gitTs, err := getGitTs()
 445         if err != nil {
 446                 return "", err
 447         }
 448         cmdName := "/bin/date"
 449         cmdArgs := []string{"-ud", "@" + string(gitTs), "+%Y%m%d%H%M%S"}
 450         date, err := exec.Command(cmdName, cmdArgs...).Output()
 451         if err != nil {
 452                 logError([]string{"There was an error running the command ", cmdName, strings.Join(cmdArgs, " "), err.Error()})
 453                 return "", fmt.Errorf("There was an error converting the datestamp for this revision")
 454         }
 455
 456         return fmt.Sprintf("%s.%s", strings.TrimSpace(prefix), strings.TrimSpace(string(date))), nil
 457 }
 458
 459 // Generates a python package version number from the git log for the current working directory
 460 func pythonVersionFromGit(prefix string) (string, error) {
 461         rv, err := rubyVersionFromGit(prefix)
 462         if err != nil {
 463                 return "", err
 464         }
 465         return rv, nil
 466 }
 467
 468 // Generates a docker image version number from the git log for the current working directory
 469 func dockerVersionFromGit() (string, error) {
 470         rv, err := gitHashFull()
 471         if err != nil {
 472                 return "", err
 473         }
 474         return rv, nil
 475 }
 476
 477 func getGitTs() (gitTs []byte, err error) {
 478         cmdArgs := []string{"log", "-n1", "--first-parent", "--max-count=1", "--format=format:%ct", "."}
 479         gitTs, err = exec.Command(theConfig.GitExecutablePath, cmdArgs...).Output()
 480         if err != nil {
 481                 logError([]string{"There was an error running the command ", theConfig.GitExecutablePath, strings.Join(cmdArgs, " "), err.Error()})
 482                 return nil, fmt.Errorf("There was an error getting the git hash for this revision")
 483         }
 484         return
 485 }
 486
 487 // Generates a timestamp from the git log for the current working directory
 488 func timestampFromGit() (string, error) {
 489         gitTs, err := getGitTs()
 490         if err != nil {
 491                 return "", err
 492         }
 493         return fmt.Sprintf("%s", strings.TrimSpace(string(gitTs))), nil
 494 }
 495
 496 func normalizeRequestedHash(hash string) (string, error) {
 497         _, err := prepareGitCheckout(hash)
 498         if err != nil {
 499                 return "", err
 500         }
 501
 502         // Get the git hash for the tree
 503         var gitHash string
 504         gitHash, err = gitHashFull()
 505         if err != nil {
 506                 return "", err
 507         }
 508
 509         return gitHash, nil
 510 }
 511
 512 func getPackageVersionsWorker(hash string) (gitHash string, goSDKTimestamp string, goSDKVersionWithoutPrefix string, pythonSDKTimestamp string, err error) {
 513         _, err = prepareGitCheckout(hash)
 514         if err != nil {
 515                 return "", "", "", "", err
 516         }
 517
 518         // Get the git hash for the tree
 519         gitHash, err = gitHashFull()
 520         if err != nil {
 521                 return "", "", "", "", err
 522         }
 523
 524         // Get the git timestamp and version string for the sdk/go directory
 525         err = os.Chdir(theConfig.DirPath + "/sdk/go")
 526         if err != nil {
 527                 goSDKTimestamp = ""
 528                 goSDKVersionWithoutPrefix = ""
 529                 err = nil
 530         } else {
 531                 goSDKTimestamp, err = timestampFromGit()
 532                 if err != nil {
 533                         return "", "", "", "", err
 534                 }
 535                 goSDKVersionWithoutPrefix, err = versionFromGit("")
 536                 if err != nil {
 537                         return "", "", "", "", err
 538                 }
 539         }
 540
 541         // Get the git timestamp and version string for the sdk/python directory
 542         err = os.Chdir(theConfig.DirPath + "/sdk/python")
 543         if err != nil {
 544                 pythonSDKTimestamp = ""
 545                 err = nil
 546         } else {
 547                 pythonSDKTimestamp, err = timestampFromGit()
 548                 if err != nil {
 549                         return "", "", "", "", err
 550                 }
 551         }
 552
 553         return
 554 }
 555
 556 func pythonSDKVersionCheck(pythonSDKTimestamp string) (err error) {
 557         var packageTimestamp string
 558         packageTimestamp, err = timestampFromGit()
 559         if err != nil {
 560                 return
 561         }
 562
 563         if pythonSDKTimestamp > packageTimestamp {
 564                 err = os.Chdir(theConfig.DirPath + "/sdk/python")
 565                 if err != nil {
 566                         return
 567                 }
 568         }
 569         return
 570 }
 571
 572 func getPackageVersions(hash string) (versions map[string]map[string]string, gitHash string, err error) {
 573         versions = make(map[string]map[string]string)
 574
 575         gitHash, goSDKTimestamp, goSDKVersionWithoutPrefix, pythonSDKTimestamp, err := getPackageVersionsWorker(hash)
 576         if err != nil {
 577                 return nil, "", err
 578         }
 579
 580         for _, p := range theConfig.Packages {
 581                 err = os.Chdir(theConfig.DirPath + "/" + p.sourceDir)
 582                 if err != nil {
 583                         // Skip those packages for which the source directory doesn't exist
 584                         // in this revision of the source tree.
 585                         err = nil
 586                         continue
 587                 }
 588                 packageName := p.packageName
 589
 590                 var packageVersion string
 591
 592                 if (p.packageVersionType == "git") || (p.packageVersionType == "go") {
 593                         packageVersion, err = versionFromGit(p.packageVersionPrefix)
 594                         if err != nil {
 595                                 return nil, "", err
 596                         }
 597                 }
 598                 if p.packageVersionType == "go" {
 599                         var packageTimestamp string
 600                         packageTimestamp, err = timestampFromGit()
 601                         if err != nil {
 602                                 return nil, "", err
 603                         }
 604
 605                         if goSDKTimestamp > packageTimestamp {
 606                                 packageVersion = p.packageVersionPrefix + goSDKVersionWithoutPrefix
 607                         }
 608                 } else if p.packageVersionType == "python" {
 609                         // Not all of our packages that use our python sdk are automatically
 610                         // getting rebuilt when sdk/python changes. Yet.
 611                         if p.packageName == "python-arvados-cwl-runner" {
 612                                 err = pythonSDKVersionCheck(pythonSDKTimestamp)
 613                                 if err != nil {
 614                                         return nil, "", err
 615                                 }
 616                         }
 617
 618                         packageVersion, err = pythonVersionFromGit(p.packageVersionPrefix)
 619                         if err != nil {
 620                                 return nil, "", err
 621                         }
 622                 } else if p.packageVersionType == "ruby" {
 623                         packageVersion, err = rubyVersionFromGit(p.packageVersionPrefix)
 624                         if err != nil {
 625                                 return nil, "", err
 626                         }
 627                 } else if p.packageVersionType == "docker" {
 628                         // the arvados/jobs image version is always the latest of the
 629                         // sdk/python and the sdk/cwl version
 630                         if p.packageName == "arvados/jobs" {
 631                                 err = pythonSDKVersionCheck(pythonSDKTimestamp)
 632                                 if err != nil {
 633                                         return nil, "", err
 634                                 }
 635                         }
 636                         packageVersion, err = dockerVersionFromGit()
 637                         if err != nil {
 638                                 return nil, "", err
 639                         }
 640                 }
 641
 642                 if versions[strings.Title(p.packageType)] == nil {
 643                         versions[strings.Title(p.packageType)] = make(map[string]string)
 644                 }
 645                 versions[strings.Title(p.packageType)][packageName] = packageVersion
 646         }
 647
 648         return
 649 }
 650
 651 func logError(m []string) {
 652         fmt.Fprintln(os.Stderr, string(marshal(logStruct{"Error", strings.Join(m, " ")})))
 653 }
 654
 655 func logNotice(m []string) {
 656         fmt.Fprintln(os.Stderr, string(marshal(logStruct{"Notice", strings.Join(m, " ")})))
 657 }
 658
 659 func marshal(message interface{}) (encoded []byte) {
 660         encoded, err := json.Marshal(message)
 661         if err != nil {
 662                 // do not call logError here because that would create an infinite loop
 663                 fmt.Fprintln(os.Stderr, "{\"Error\": \"Unable to marshal message into json:", message, "\"}")
 664                 return nil
 665         }
 666         return
 667 }
 668
 669 func marshalAndWrite(w io.Writer, message interface{}) {
 670         b := marshal(message)
 671         if b == nil {
 672                 errorMessage := "{\n\"Error\": \"Unspecified error\"\n}"
 673                 _, err := io.WriteString(w, errorMessage)
 674                 if err != nil {
 675                         // do not call logError (it calls marshal and that function has already failed at this point)
 676                         fmt.Fprintln(os.Stderr, "{\"Error\": \"Unable to write message to client\"}")
 677                 }
 678         } else {
 679                 _, err := w.Write(b)
 680                 if err != nil {
 681                         logError([]string{"Unable to write message to client:", string(b)})
 682                 }
 683         }
 684 }
 685
 686 func packageVersionHandler(w http.ResponseWriter, r *http.Request) {
 687         start := time.Now()
 688         w.Header().Set("Content-Type", "application/json; charset=utf-8")
 689
 690         var packageVersions map[string]map[string]string
 691         var cached bool
 692
 693         // Sanity check the input RequestHash
 694         match, err := regexp.MatchString("^([a-z0-9]+|)$", r.URL.Path[11:])
 695         if err != nil {
 696                 m := logStruct{"Error", "Error matching RequestHash"}
 697                 marshalAndWrite(w, m)
 698                 return
 699         }
 700         if !match {
 701                 m := logStruct{"Error", "Invalid RequestHash"}
 702                 marshalAndWrite(w, m)
 703                 return
 704         }
 705
 706         hash := r.URL.Path[11:]
 707
 708         // Empty hash or non-standard hash length? Normalize it.
 709         if len(hash) != 7 && len(hash) != 40 {
 710                 hash, err = normalizeRequestedHash(hash)
 711                 if err != nil {
 712                         m := logStruct{"Error", err.Error()}
 713                         marshalAndWrite(w, m)
 714                         return
 715                 }
 716         }
 717
 718         var gitHash string
 719         rs, err := lookupInCache(hash)
 720         if err == nil {
 721                 packageVersions = rs.Versions
 722                 gitHash = rs.GitHash
 723                 cached = true
 724         } else {
 725                 packageVersions, gitHash, err = getPackageVersions(hash)
 726                 if err != nil {
 727                         m := logStruct{"Error", err.Error()}
 728                         marshalAndWrite(w, m)
 729                         return
 730                 }
 731                 m := returnStruct{"", gitHash, packageVersions, true, ""}
 732                 err = writeToCache(hash, m)
 733                 if err != nil {
 734                         logError([]string{"Unable to save entry in cache directory", theConfig.CacheDirPath})
 735                 }
 736                 cached = false
 737         }
 738
 739         m := returnStruct{hash, gitHash, packageVersions, cached, fmt.Sprintf("%v", time.Since(start))}
 740         marshalAndWrite(w, m)
 741 }
 742
 743 func aboutHandler(w http.ResponseWriter, r *http.Request) {
 744         w.Header().Set("Content-Type", "application/json; charset=utf-8")
 745         m := aboutStruct{"Arvados Version Server", "0.1", "https://arvados.org"}
 746         marshalAndWrite(w, m)
 747 }
 748
 749 func helpHandler(w http.ResponseWriter, r *http.Request) {
 750         w.Header().Set("Content-Type", "application/json; charset=utf-8")
 751         m := helpStruct{"GET /v1/commit/ or GET /v1/commit/git-commit or GET /v1/about or GET /v1/help"}
 752         marshalAndWrite(w, m)
 753 }
 754
 755 func parseFlags() (configPath *string) {
 756
 757         flags := flag.NewFlagSet("arvados-version-server", flag.ExitOnError)
 758         flags.Usage = func() { usage(flags) }
 759
 760         configPath = flags.String(
 761                 "config",
 762                 defaultConfigPath,
 763                 "`path` to YAML configuration file")
 764
 765         // Parse args; omit the first arg which is the command name
 766         err := flags.Parse(os.Args[1:])
 767         if err != nil {
 768                 logError([]string{"Unable to parse command line arguments:", err.Error()})
 769                 os.Exit(1)
 770         }
 771
 772         return
 773 }
 774
 775 func main() {
 776         err := os.Setenv("TZ", "UTC")
 777         if err != nil {
 778                 logError([]string{"Error setting environment variable:", err.Error()})
 779                 os.Exit(1)
 780         }
 781
 782         configPath := parseFlags()
 783
 784         err = readConfig(&theConfig, *configPath, defaultConfigPath)
 785         if err != nil {
 786                 logError([]string{"Unable to start Arvados Version Server:", err.Error()})
 787                 os.Exit(1)
 788         }
 789
 790         theConfig.Packages = loadPackages()
 791
 792         if theConfig.DirPath == "" {
 793                 theConfig.DirPath = "/tmp/arvados-version-server-checkout"
 794         }
 795
 796         if theConfig.CacheDirPath == "" {
 797                 theConfig.CacheDirPath = "/tmp/arvados-version-server-cache"
 798         }
 799
 800         if theConfig.GitExecutablePath == "" {
 801                 theConfig.GitExecutablePath = "/usr/bin/git"
 802         }
 803
 804         if theConfig.ListenPort == "" {
 805                 theConfig.ListenPort = "80"
 806         }
 807
 808         http.HandleFunc("/v1/commit/", packageVersionHandler)
 809         http.HandleFunc("/v1/about", aboutHandler)
 810         http.HandleFunc("/v1/help", helpHandler)
 811         http.HandleFunc("/v1", helpHandler)
 812         http.HandleFunc("/", helpHandler)
 813         logNotice([]string{"Arvados Version Server listening on port", theConfig.ListenPort})
 814
 815         listener, err = net.Listen("tcp", ":"+theConfig.ListenPort)
 816
 817         if err != nil {
 818                 logError([]string{"Unable to start Arvados Version Server:", err.Error()})
 819                 os.Exit(1)
 820         }
 821
 822         // Shut down the server gracefully (by closing the listener)
 823         // if SIGTERM is received.
 824         term := make(chan os.Signal, 1)
 825         go func(sig <-chan os.Signal) {
 826                 <-sig
 827                 logError([]string{"caught signal"})
 828                 _ = listener.Close()
 829         }(term)
 830         signal.Notify(term, syscall.SIGTERM)
 831         signal.Notify(term, syscall.SIGINT)
 832
 833         // Start serving requests.
 834         _ = http.Serve(listener, nil)
 835         // http.Serve returns an error when it gets the term or int signal
 836
 837         logNotice([]string{"Arvados Version Server shutting down"})
 838
 839 }
 840
 841 func readConfig(cfg interface{}, path string, defaultConfigPath string) error {
 842         err := config.LoadFile(cfg, path)
 843         if err != nil && os.IsNotExist(err) && path == defaultConfigPath {
 844                 logNotice([]string{"Config not specified. Continue with default configuration."})
 845                 err = nil
 846         }
 847         return err
 848 }