From: Ward Vandewege Date: Fri, 6 Nov 2020 18:22:22 +0000 (-0500) Subject: Upgrade the helm charts to Arvados 2.1.0. X-Git-Url: https://git.arvados.org/arvados-k8s.git/commitdiff_plain/7f7c5c213af259bb660644949ce19a0d383e0bf2 Upgrade the helm charts to Arvados 2.1.0. This removes the (deprecated) SSO server. The minimum Arvados version that will work with the chart is now 2.1.0. closes #17095 Arvados-DCO-1.1-Signed-off-by: Ward Vandewege --- diff --git a/charts/arvados/config/api-server/90-init-db.sh b/charts/arvados/config/api-server/90-init-db.sh index cc16372..c488c2e 100644 --- a/charts/arvados/config/api-server/90-init-db.sh +++ b/charts/arvados/config/api-server/90-init-db.sh @@ -22,6 +22,8 @@ prepare_database() { if [[ -f "/create-workbench-api-client.rb" ]]; then # This is the API server cd /var/www/arvados-api/current + # The script/rails command in the Arvados 2.1.0 release has an incorrect require path. + sed -i 's|rails/commands/server|rails/command|' script/rails bundle exec script/create_superuser_token.rb {{ .Values.superUserSecret }} cd script bundle exec get_anonymous_user_token.rb -t {{ .Values.anonymousUserSecret }} || true diff --git a/charts/arvados/config/api-server/database.yml b/charts/arvados/config/api-server/database.yml deleted file mode 100644 index 8b1ad41..0000000 --- a/charts/arvados/config/api-server/database.yml +++ /dev/null @@ -1,33 +0,0 @@ -# Copyright (C) The Arvados Authors. All rights reserved. -# -# SPDX-License-Identifier: Apache-2.0 - -production: - adapter: postgresql - encoding: utf8 - database: arvados_production - username: arvados - password: pw - host: arvados-postgres - template: template0 - -development: - adapter: postgresql - encoding: utf8 - database: arvados_sso_development - username: arvados_sso - password: xxxxxxxx - host: localhost - template: template0 - -# Warning: The database defined as "test" will be erased and -# re-generated from your development database when you run "rake". -# Do not set this db to the same as development or production. -test: - adapter: postgresql - encoding: utf8 - database: arvados_sso_test - username: arvados_sso - password: xxxxxxxx - host: localhost - template: template0 diff --git a/charts/arvados/config/config.yml b/charts/arvados/config/config.yml index 2896474..305df3d 100644 --- a/charts/arvados/config/config.yml +++ b/charts/arvados/config/config.yml @@ -25,8 +25,12 @@ Clusters: DNSServerConfDir: /tmp DNSServerConfTemplate: /var/www/arvados-api/current/config/unbound.template Login: - ProviderAppID: arvados-server - ProviderAppSecret: app_secret + Test: + Enable: true + Users: + admin: + Email: {{ .Values.adminUserEmail }} + Password: {{ .Values.adminUserPassword }} PostgreSQL: Connection: dbname: arvados_production @@ -79,8 +83,6 @@ Clusters: InternalURLs: http://arvados-keep-store-0.arvados-keep-store:25107: {} http://arvados-keep-store-1.arvados-keep-store:25107: {} - SSO: - ExternalURL: https://{{ .Values.externalIP }}:3002 WebDAV: ExternalURL: https://{{ .Values.externalIP }}:9002 InternalURLs: @@ -99,3 +101,4 @@ Clusters: AutoAdminFirstUser: true AutoSetupNewUsers: true NewUsersAreActive: true + AnonymousUserToken: {{ .Values.anonymousUserSecret }} diff --git a/charts/arvados/config/postgres/create_dbs.sh b/charts/arvados/config/postgres/create_dbs.sh index d7b419b..939bd93 100644 --- a/charts/arvados/config/postgres/create_dbs.sh +++ b/charts/arvados/config/postgres/create_dbs.sh @@ -17,6 +17,5 @@ EOSQL EOSQL } -create_user_and_database arvados_sso_production arvados_sso pw create_user_and_database arvados_production arvados pw diff --git a/charts/arvados/config/sso/90-init-db.sh b/charts/arvados/config/sso/90-init-db.sh deleted file mode 100644 index 0515dd6..0000000 --- a/charts/arvados/config/sso/90-init-db.sh +++ /dev/null @@ -1,24 +0,0 @@ -#!/bin/bash -# Copyright (C) The Arvados Authors. All rights reserved. -# -# SPDX-License-Identifier: Apache-2.0 - -set -e - -prepare_database() { - RAILSPKG_DATABASE_LOAD_TASK="$1" - DB_MIGRATE_STATUS=$($COMMAND_PREFIX bundle exec rake db:migrate:status 2>&1 || true) - if echo "$DB_MIGRATE_STATUS" | grep -qF 'Schema migrations table does not exist yet.'; then - # The database exists, but the migrations table doesn't. - bundle exec rake "$RAILSPKG_DATABASE_LOAD_TASK" db:seed - elif echo "$DB_MIGRATE_STATUS" | grep -q '^database: '; then - bundle exec rake db:migrate - elif echo "$DB_MIGRATE_STATUS" | grep -q 'database .* does not exist'; then - bundle exec rake db:setup - else - echo "Warning: Database is not ready to set up." >&2 - exit 1 - fi -} - -prepare_database "db:schema:load" diff --git a/charts/arvados/config/sso/92-init-client.sh b/charts/arvados/config/sso/92-init-client.sh deleted file mode 100644 index 9acf827..0000000 --- a/charts/arvados/config/sso/92-init-client.sh +++ /dev/null @@ -1,7 +0,0 @@ -#!/bin/bash -# Copyright (C) The Arvados Authors. All rights reserved. -# -# SPDX-License-Identifier: Apache-2.0 - -set -e -bundle exec rails runner /init-client.rb diff --git a/charts/arvados/config/sso/application.yml b/charts/arvados/config/sso/application.yml deleted file mode 100644 index 3f92c81..0000000 --- a/charts/arvados/config/sso/application.yml +++ /dev/null @@ -1,94 +0,0 @@ -# Copyright (C) The Arvados Authors. All rights reserved. -# -# SPDX-License-Identifier: Apache-2.0 - -# Copy this file to application.yml and edit to suit. -# -# Consult application.default.yml for the full list of configuration -# settings. -# -# The order of precedence is: -# 1. config/environments/{RAILS_ENV}.rb (deprecated) -# 2. Section in application.yml corresponding to RAILS_ENV (e.g., development) -# 3. Section in application.yml called "common" -# 4. Section in application.default.yml corresponding to RAILS_ENV -# 5. Section in application.default.yml called "common" - -common: - uuid_prefix: abcde - secret_token: zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz - - # The site name that will be used in text such as "Sign in to site_title" - site_title: Arvados - - # After logging in, the title and URL of the link that will be presented to - # the user as the default destination on the welcome page. - default_link_title: Arvados - default_link_url: "http://localhost:3000" - - ### - ### Local account configuration. This is enabled if neither - ### google_oauth2 or LDAP are enabled below. - ### - # If true, allow new creation of new accounts in the SSO server's internal - # user database. - allow_account_registration: true - - # If true, send an email confirmation before activating new accounts in the - # SSO server's internal user database. - require_email_confirmation: false - - - ### - ### Google+ OAuth2 authentication. - ### - # Google API tokens required for OAuth2 login. - # - # See https://github.com/zquestz/omniauth-google-oauth2 - # - # and https://developers.google.com/accounts/docs/OAuth2 - google_oauth2_client_id: false - google_oauth2_client_secret: false - - # Set this to your OpenId 2.0 realm to enable migration from Google OpenId - # 2.0 to Google OAuth2 OpenId Connect (Google will provide OpenId 2.0 user - # identifiers via the openid.realm parameter in the OAuth2 flow until 2017). - google_openid_realm: false - - - ### - ### LDAP authentication. - ### - # - # If you want to use LDAP, you need to provide - # the following set of fields under the use_ldap key. - # - # If 'email_domain' field is set, it will be stripped from the email address - # entered by the user prior attempting LDAP binding on 'uid'. This supports - # the case where it is not possible to look up 'bob@example.com' but instead - # must be looked up as 'uid=bob'. - # - # If it is possible to look up the email address directly (for example - # setting "uid: mail") you should prefer that and leave 'email_domain' unset. - # - # If 'username' is set, this specifies the LDAP field that will be propagated - # to the "username" field in the users table. This should be a - # posix-compatible username (which may be different from the username part of - # the email address.) - # - # Provide 'bind_dn' and 'password' if your LDAP server requires - # a login before authenticating a user. - # - # use_ldap: - # title: Example LDAP - # host: ldap.example.com - # port: 636 - # method: ssl - # base: "ou=Users, dc=example, dc=com" - # uid: uid - # username: uid - # #email_domain: example.com - # #bind_dn: "some_user" - # #password: "some_password" - # - use_ldap: false diff --git a/charts/arvados/config/sso/database.yml b/charts/arvados/config/sso/database.yml deleted file mode 100644 index 7045747..0000000 --- a/charts/arvados/config/sso/database.yml +++ /dev/null @@ -1,33 +0,0 @@ -# Copyright (C) The Arvados Authors. All rights reserved. -# -# SPDX-License-Identifier: Apache-2.0 - -production: - adapter: postgresql - encoding: utf8 - database: arvados_sso_production - username: arvados_sso - password: pw - host: arvados-postgres - template: template0 - -development: - adapter: postgresql - encoding: utf8 - database: arvados_sso_development - username: arvados_sso - password: xxxxxxxx - host: localhost - template: template0 - -# Warning: The database defined as "test" will be erased and -# re-generated from your development database when you run "rake". -# Do not set this db to the same as development or production. -test: - adapter: postgresql - encoding: utf8 - database: arvados_sso_test - username: arvados_sso - password: xxxxxxxx - host: localhost - template: template0 diff --git a/charts/arvados/config/sso/init-client.rb b/charts/arvados/config/sso/init-client.rb deleted file mode 100644 index 508106d..0000000 --- a/charts/arvados/config/sso/init-client.rb +++ /dev/null @@ -1,13 +0,0 @@ -# Copyright (C) The Arvados Authors. All rights reserved. -# -# SPDX-License-Identifier: Apache-2.0 - -c = Client.new -c.name = "joshid" -c.app_id = "arvados-server" -c.app_secret = "app_secret" -c.save! - -User.find_or_create_by_email(email: "{{ .Values.adminUserEmail }}") do |user| - user.password = "{{ .Values.adminUserPassword }}" -end diff --git a/charts/arvados/config/sso/nginx.conf b/charts/arvados/config/sso/nginx.conf deleted file mode 100644 index 76d6530..0000000 --- a/charts/arvados/config/sso/nginx.conf +++ /dev/null @@ -1,51 +0,0 @@ -# Copyright (C) The Arvados Authors. All rights reserved. -# -# SPDX-License-Identifier: Apache-2.0 - -server { - listen 127.0.0.1:9000; - server_name localhost-sso; - - root /var/www/arvados-sso/current/public; - index index.html index.htm index.php; - - passenger_enabled on; - # If you're using RVM, uncomment the line below. - passenger_ruby /usr/local/rvm/wrappers/default/ruby; - - # `client_max_body_size` should match the corresponding setting in - # the API server's Nginx configuration. - client_max_body_size 128m; -} - -upstream sso { - server 127.0.0.1:9000 fail_timeout=10s; -} - -proxy_http_version 1.1; - -server { - listen 0.0.0.0:3002 ssl; - server_name public-sso; - - ssl on; - ssl_certificate /etc/ssl/certs/sso.pem; - ssl_certificate_key /etc/ssl/private/sso.key; - - index index.html index.htm index.php; - # `client_max_body_size` should match the corresponding setting in - # the API server's Nginx configuration. - client_max_body_size 128m; - - location / { - proxy_pass http://sso; - proxy_redirect off; - proxy_connect_timeout 90s; - proxy_read_timeout 300s; - - proxy_set_header X-Forwarded-Proto https; - proxy_set_header Host $http_host; - proxy_set_header X-Real-IP $remote_addr; - proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; - } -} diff --git a/charts/arvados/config/workbench/database.yml b/charts/arvados/config/workbench/database.yml deleted file mode 100644 index b2e59cf..0000000 --- a/charts/arvados/config/workbench/database.yml +++ /dev/null @@ -1,51 +0,0 @@ -# Copyright (C) The Arvados Authors. All rights reserved. -# -# SPDX-License-Identifier: Apache-2.0 - -server { - listen 127.0.0.1:9000; - server_name localhost-workbench; - - root /var/www/arvados-workbench/current/public; - index index.html index.htm index.php; - - passenger_enabled on; - # If you're using RVM, uncomment the line below. - passenger_ruby /usr/local/rvm/wrappers/default/ruby; - - # `client_max_body_size` should match the corresponding setting in - # the API server's Nginx configuration. - client_max_body_size 128m; -} - -upstream workbench { - server 127.0.0.1:9000 fail_timeout=10s; -} - -proxy_http_version 1.1; - -server { - listen 0.0.0.0:443 ssl; - server_name public-workbench; - - ssl on; - ssl_certificate /etc/ssl/certs/workbench.pem; - ssl_certificate_key /etc/ssl/private/workbench.key; - - index index.html index.htm index.php; - # `client_max_body_size` should match the corresponding setting in - # the API server's Nginx configuration. - client_max_body_size 128m; - - location / { - proxy_pass http://workbench; - proxy_redirect off; - proxy_connect_timeout 90s; - proxy_read_timeout 300s; - - proxy_set_header X-Forwarded-Proto https; - proxy_set_header Host $http_host; - proxy_set_header X-Real-IP $remote_addr; - proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; - } -} diff --git a/charts/arvados/minikube-external-ip.sh b/charts/arvados/minikube-external-ip.sh index 5bdccd9..c204fb2 100755 --- a/charts/arvados/minikube-external-ip.sh +++ b/charts/arvados/minikube-external-ip.sh @@ -1,5 +1,9 @@ #!/bin/bash +# Copyright (C) The Arvados Authors. All rights reserved. +# +# SPDX-License-Identifier: Apache-2.0 + EXTERNAL_IP=$1 if [[ -z "$EXTERNAL_IP" ]]; then @@ -15,7 +19,6 @@ fi kubectl patch service arvados-api-server -p "{\"spec\": {\"type\": \"LoadBalancer\", \"externalIPs\":[\"$EXTERNAL_IP\"]}}" kubectl patch service arvados-keep-proxy -p "{\"spec\": {\"type\": \"LoadBalancer\", \"externalIPs\":[\"$EXTERNAL_IP\"]}}" kubectl patch service arvados-keep-web -p "{\"spec\": {\"type\": \"LoadBalancer\", \"externalIPs\":[\"$EXTERNAL_IP\"]}}" -kubectl patch service arvados-sso -p "{\"spec\": {\"type\": \"LoadBalancer\", \"externalIPs\":[\"$EXTERNAL_IP\"]}}" kubectl patch service arvados-workbench -p "{\"spec\": {\"type\": \"LoadBalancer\", \"externalIPs\":[\"$EXTERNAL_IP\"]}}" kubectl patch service arvados-ws -p "{\"spec\": {\"type\": \"LoadBalancer\", \"externalIPs\":[\"$EXTERNAL_IP\"]}}" diff --git a/charts/arvados/templates/api-server-deployment.yaml b/charts/arvados/templates/api-server-deployment.yaml index 0b42ffa..aa7a621 100644 --- a/charts/arvados/templates/api-server-deployment.yaml +++ b/charts/arvados/templates/api-server-deployment.yaml @@ -37,9 +37,6 @@ spec: - name: api-server-configmap mountPath: /init-scripts/90-init-db.sh subPath: 90-init-db.sh - - name: api-server-configmap - mountPath: /etc/arvados/api/database.yml - subPath: database.yml - name: etc-configmap mountPath: /etc/arvados/config.yml subPath: config.yml diff --git a/charts/arvados/templates/sso-configmap.yaml b/charts/arvados/templates/sso-configmap.yaml deleted file mode 100644 index c8fb70d..0000000 --- a/charts/arvados/templates/sso-configmap.yaml +++ /dev/null @@ -1,15 +0,0 @@ -# Copyright (C) The Arvados Authors. All rights reserved. -# -# SPDX-License-Identifier: Apache-2.0 - -apiVersion: v1 -kind: ConfigMap -metadata: - name: arvados-sso-configmap - labels: - app: {{ template "arvados.name" . }} - chart: {{ template "arvados.chart" . }} - release: {{ .Release.Name }} - heritage: {{ .Release.Service }} -data: -{{ (tpl (.Files.Glob "config/sso/*").AsConfig . ) | indent 2 }} diff --git a/charts/arvados/templates/sso-deployment.yaml b/charts/arvados/templates/sso-deployment.yaml deleted file mode 100644 index b5390e8..0000000 --- a/charts/arvados/templates/sso-deployment.yaml +++ /dev/null @@ -1,68 +0,0 @@ -# Copyright (C) The Arvados Authors. All rights reserved. -# -# SPDX-License-Identifier: Apache-2.0 - -apiVersion: apps/v1 -kind: Deployment -metadata: - name: "arvados-sso" - labels: - app: arvados-sso - chart: {{ template "arvados.chart" . }} - release: {{ .Release.Name }} - heritage: {{ .Release.Service }} -spec: - selector: - matchLabels: - app: arvados-sso - release: {{ .Release.Name }} - template: - metadata: - labels: - app: arvados-sso - release: {{ .Release.Name }} - spec: - containers: - - name: {{ .Chart.Name }} - image: "cure/arvados-rails-runtime" - imagePullPolicy: {{ .Values.image.pullPolicy }} - command: - - "sh" - - "-c" - - "install /init-scripts/*.sh /etc/my_init.d && /usr/local/bin/bootstrap.sh arvados-sso-server={{ .Values.arvados.versions.distribution.arvadosSSOServer }} && cd /var/www/arvados-sso/current && exec /sbin/my_init" - env: - - name: RAILS_ENV - value: "production" - volumeMounts: - - name: sso-configmap - mountPath: /etc/arvados/sso/database.yml - subPath: database.yml - - name: sso-configmap - mountPath: /etc/arvados/sso/application.yml - subPath: application.yml - - name: sso-configmap - mountPath: /etc/nginx/sites-enabled/sso.conf - subPath: nginx.conf - - name: sso-configmap - mountPath: /init-client.rb - subPath: init-client.rb - - name: sso-configmap - mountPath: /init-scripts/90-init-db.sh - subPath: 90-init-db.sh - - name: sso-configmap - mountPath: /init-scripts/92-init-client.sh - subPath: 92-init-client.sh - - name: ssl-configmap - mountPath: /etc/ssl/certs/sso.pem - subPath: cert - - name: ssl-configmap - mountPath: /etc/ssl/private/sso.key - subPath: key - - volumes: - - name: sso-configmap - configMap: - name: arvados-sso-configmap - - name: ssl-configmap - configMap: - name: ssl-configmap diff --git a/charts/arvados/templates/sso-service.yaml b/charts/arvados/templates/sso-service.yaml deleted file mode 100644 index 31f5e8d..0000000 --- a/charts/arvados/templates/sso-service.yaml +++ /dev/null @@ -1,27 +0,0 @@ -# Copyright (C) The Arvados Authors. All rights reserved. -# -# SPDX-License-Identifier: Apache-2.0 - -apiVersion: v1 -kind: Service -metadata: - name: arvados-sso - labels: - app: {{ template "arvados.name" . }} - chart: {{ template "arvados.chart" . }} - release: {{ .Release.Name }} - heritage: {{ .Release.Service }} - annotations: - {{- range $key, $value := .Values.loadBalancer.annotations }} - {{ $key }}: {{ $value | quote }} - {{- end }} -spec: - type: LoadBalancer - loadBalancerIP: {{ required "A valid externalIP is required!" .Values.externalIP }} - ports: - - name: http - port: 3002 - targetPort: 3002 - protocol: TCP - selector: - app: arvados-sso diff --git a/charts/arvados/values.yaml b/charts/arvados/values.yaml index 1336eed..569eb95 100644 --- a/charts/arvados/values.yaml +++ b/charts/arvados/values.yaml @@ -47,28 +47,27 @@ uuidPrefix: vwxyz arvados: versions: distribution: - arvadosApiServer: 2.0.2-1 - arvadosController: 2.0.2-1 - arvadosGitHttpd: 2.0.2-1 - arvadosHealth: 2.0.2-1 - arvadosWorkbench: 2.0.2-1 - arvadosWorkbench2: 2.0.2-1 - arvadosWs: 2.0.2-1 - arvadosSSOServer: 1.1.4-1 - crunchDispatchSlurm: 2.0.2-1 - crunchRun: 2.0.2-1 - crunchRunner: 2.0.2-1 - keepBalance: 2.0.2-1 - keepStore: 2.0.2-1 - keepWeb: 2.0.2-1 - keepProxy: 2.1.0.dev20200422013447-1 - libPamArvados: 2.0.2-1 - pythonArvadosFuse: 2.0.2-1 - pythonArvadosPythonClient: 2.0.2-1 + arvadosApiServer: 2.1.0-1 + arvadosController: 2.1.0-1 + arvadosGitHttpd: 2.1.0-1 + arvadosHealth: 2.1.0-1 + arvadosWorkbench: 2.1.0-1 + arvadosWorkbench2: 2.1.0-1 + arvadosWs: 2.1.0-1 + crunchDispatchSlurm: 2.1.0-1 + crunchRun: 2.1.0-1 + crunchRunner: 2.1.0-1 + keepBalance: 2.1.0-1 + keepStore: 2.1.0-1 + keepWeb: 2.1.0-1 + keepProxy: 2.1.0-1 + libPamArvados: 2.1.0-1 + pythonArvadosFuse: 2.1.0-1 + pythonArvadosPythonClient: 2.1.0-1 gem: - arvados: 2.0.2 - arvadosCLI: 2.0.2 - arvadosLoginSync: 2.0.2 + arvados: 2.1.0 + arvadosCLI: 2.1.0 + arvadosLoginSync: 2.1.0 # A custom bundle of CA certificates to use. # Useful for corporate networks with TLS proxies. diff --git a/dockerfiles/Dockerfile.rails-runtime b/dockerfiles/Dockerfile.rails-runtime index 90857ec..2623afc 100644 --- a/dockerfiles/Dockerfile.rails-runtime +++ b/dockerfiles/Dockerfile.rails-runtime @@ -2,7 +2,7 @@ # # SPDX-License-Identifier: Apache-2.0 -FROM phusion/passenger-ruby23 +FROM phusion/passenger-ruby25 RUN apt-get update && apt-get install -y gnupg2 @@ -20,8 +20,8 @@ RUN apt-get update && apt-get install -qqy tzdata # preinstall latest arvados rails packages, so that we have (most of) the gems # baked into this docker image. Then remove them so that this image is generic. # The bundles will remain installed. -RUN apt-get install -qqy arvados-workbench arvados-sso-server arvados-api-server -RUN apt-get remove -qqy arvados-workbench arvados-sso-server arvados-api-server arvados-server +RUN apt-get install -qqy arvados-workbench arvados-api-server +RUN apt-get remove -qqy arvados-workbench arvados-api-server arvados-server # put nginx.conf back into the state it needs to be RUN sed -i 's/#daemon off;/daemon off;/' /etc/nginx/nginx.conf diff --git a/dockerfiles/bootstrap.sh b/dockerfiles/bootstrap.sh index cfdc8c4..784e4d1 100755 --- a/dockerfiles/bootstrap.sh +++ b/dockerfiles/bootstrap.sh @@ -9,7 +9,7 @@ if [[ "$1" == "" ]]; then exit 1 fi -if [[ "$@" =~ "arvados-workbench=" ]] || [[ "$@" =~ "arvados-sso-server=" ]] || [[ "$@" =~ "arvados-api-server=" ]]; then +if [[ "$@" =~ "arvados-workbench=" ]] || [[ "$@" =~ "arvados-api-server=" ]]; then RESET_NGINX_DAEMON_FLAG=true else RESET_NGINX_DAEMON_FLAG=false