From: Peter Amstutz Date: Mon, 1 May 2017 17:29:49 +0000 (-0400) Subject: Merge branch 'master' into origin-8019-crunchrun-log-throttle X-Git-Tag: 1.1.0~267^2~5 X-Git-Url: https://git.arvados.org/arvados.git/commitdiff_plain/5d00ecb0932f86e4d2aced3d9258b96522ef38bd?hp=-c Merge branch 'master' into origin-8019-crunchrun-log-throttle --- 5d00ecb0932f86e4d2aced3d9258b96522ef38bd diff --combined services/api/app/controllers/arvados/v1/schema_controller.rb index 07a581f4c2,11269d2556..e1f4ca5770 --- a/services/api/app/controllers/arvados/v1/schema_controller.rb +++ b/services/api/app/controllers/arvados/v1/schema_controller.rb @@@ -38,13 -38,7 +38,14 @@@ class Arvados::V1::SchemaController < A blobSignatureTtl: Rails.application.config.blob_signature_ttl, maxRequestSize: Rails.application.config.max_request_size, dockerImageFormats: Rails.application.config.docker_image_formats, + crunchLogBytesPerEvent: Rails.application.config.crunch_log_bytes_per_event, + crunchLogSecondsBetweenEvents: Rails.application.config.crunch_log_seconds_between_events, + crunchLogThrottlePeriod: Rails.application.config.crunch_log_throttle_period, + crunchLogThrottleBytes: Rails.application.config.crunch_log_throttle_bytes, + crunchLogThrottleLines: Rails.application.config.crunch_log_throttle_lines, + crunchLimitLogBytesPerJob: Rails.application.config.crunch_limit_log_bytes_per_job, + crunchLogPartialLineThrottlePeriod: Rails.application.config.crunch_log_partial_line_throttle_period, + websocketUrl: Rails.application.config.websocket_address, parameters: { alt: { type: "string", @@@ -90,12 -84,6 +91,6 @@@ resources: {} } - if Rails.application.config.websocket_address - discovery[:websocketUrl] = Rails.application.config.websocket_address - elsif ENV['ARVADOS_WEBSOCKETS'] - discovery[:websocketUrl] = root_url.sub(/^http/, 'ws') + "websocket" - end - ActiveRecord::Base.descendants.reject(&:abstract_class?).each do |k| begin ctl_class = "Arvados::V1::#{k.to_s.pluralize}Controller".constantize diff --combined services/api/config/application.default.yml index db87265b7b,85955be4e2..8118914ec9 --- a/services/api/config/application.default.yml +++ b/services/api/config/application.default.yml @@@ -46,28 -46,16 +46,16 @@@ common # to log in. workbench_address: false - # The ARVADOS_WEBSOCKETS environment variable determines whether to - # serve http, websockets, or both. + # Client-facing URI for websocket service. Nginx should be + # configured to proxy this URI to arvados-ws; see + # http://doc.arvados.org/install/install-ws.html # - # If ARVADOS_WEBSOCKETS="true", http and websockets are both served - # from the same process. + # If websocket_address is false (which is the default), no websocket + # server will be advertised to clients. This configuration is not + # supported. # - # If ARVADOS_WEBSOCKETS="ws-only", only websockets is served. - # - # If ARVADOS_WEBSOCKETS="false" or not set at all, only http is - # served. In this case, you should have a separate process serving - # websockets, and the address of that service should be given here - # as websocket_address. - # - # If websocket_address is false (which is the default), the - # discovery document will tell clients to use the current server as - # the websocket service, or (if the current server does not have - # websockets enabled) not to use websockets at all. - # - # Example: Clients will connect to the specified endpoint. - #websocket_address: wss://127.0.0.1:3333/websocket - # Default: Clients will connect to this server if it's running - # websockets, otherwise none at all. + # Example: + #websocket_address: wss://ws.zzzzz.arvadosapi.com/websocket websocket_address: false # Maximum number of websocket connections allowed @@@ -279,8 -267,6 +267,8 @@@ # silenced by throttling are not counted against this total. crunch_limit_log_bytes_per_job: 67108864 + crunch_log_partial_line_throttle_period: 5 + # Attributes to suppress in events and audit logs. Notably, # specifying ["manifest_text"] here typically makes the database # smaller and faster. @@@ -415,6 -401,8 +403,6 @@@ # "git log". source_version: false - crunch_log_partial_line_throttle_period: 5 - # Enable asynchronous permission graph rebuild. Must run # script/permission-updater.rb as a separate process. When the permission # cache is invalidated, the background process will update the permission @@@ -441,7 -429,6 +429,6 @@@ development action_mailer.perform_deliveries: false active_support.deprecation: :log action_dispatch.best_standards_support: :builtin - active_record.mass_assignment_sanitizer: :strict active_record.auto_explain_threshold_in_seconds: 0.5 assets.compress: false assets.debug: true @@@ -451,7 -438,7 +438,7 @@@ production cache_classes: true consider_all_requests_local: false action_controller.perform_caching: true - serve_static_assets: false + serve_static_files: false assets.compress: true assets.compile: false assets.digest: true @@@ -459,7 -446,7 +446,7 @@@ test: force_ssl: false cache_classes: true - serve_static_assets: true + serve_static_files: true static_cache_control: public, max-age=3600 whiny_nils: true consider_all_requests_local: true @@@ -468,7 -455,6 +455,6 @@@ action_controller.allow_forgery_protection: false action_mailer.delivery_method: :test active_support.deprecation: :stderr - active_record.mass_assignment_sanitizer: :strict uuid_prefix: zzzzz sso_app_id: arvados-server sso_app_secret: <%= rand(2**512).to_s(36) %> @@@ -479,6 -465,6 +465,6 @@@ workbench_address: https://localhost:3001/ git_repositories_dir: <%= Rails.root.join 'tmp', 'git', 'test' %> git_internal_dir: <%= Rails.root.join 'tmp', 'internal.git' %> - websocket_address: <% if ENV['ARVADOS_TEST_EXPERIMENTAL_WS'] %>"wss://0.0.0.0:<%= ENV['ARVADOS_TEST_WSS_PORT'] %>/websocket"<% else %>false<% end %> + websocket_address: "wss://0.0.0.0:<%= ENV['ARVADOS_TEST_WSS_PORT'] %>/websocket" trash_sweep_interval: -1 docker_image_formats: ["v1"] diff --combined services/crunch-run/crunchrun.go index c9c52ee02f,d1c24a5acb..812525db69 --- a/services/crunch-run/crunchrun.go +++ b/services/crunch-run/crunchrun.go @@@ -145,6 -145,7 +145,7 @@@ type ContainerRunner struct HostOutputDir string CleanupTempDir []string Binds []string + Volumes map[string]struct{} OutputPDH *string SigChan chan os.Signal ArvMountExit chan error @@@ -336,6 -337,7 +337,7 @@@ func (runner *ContainerRunner) SetupMou collectionPaths := []string{} runner.Binds = nil + runner.Volumes = make(map[string]struct{}) needCertMount := true var binds []string @@@ -445,7 -447,7 +447,7 @@@ runner.Binds = append(runner.Binds, fmt.Sprintf("%s:%s", runner.HostOutputDir, bind)) case mnt.Kind == "tmp": - runner.Binds = append(runner.Binds, bind) + runner.Volumes[bind] = struct{}{} case mnt.Kind == "json": jsondata, err := json.Marshal(mnt.Content) @@@ -631,12 -633,11 +633,12 @@@ func (runner *ContainerRunner) LogNodeI // Get and save the raw JSON container record from the API server func (runner *ContainerRunner) LogContainerRecord() (err error) { w := &ArvLogWriter{ - runner.ArvClient, - runner.Container.UUID, - "container", - runner.LogCollection.Open("container.json"), + ArvClient: runner.ArvClient, + UUID: runner.Container.UUID, + loggingStream: "container", + writeCloser: runner.LogCollection.Open("container.json"), } + // Get Container record JSON from the API Server reader, err := runner.ArvClient.CallRaw("GET", "containers", runner.Container.UUID, "", nil) if err != nil { @@@ -794,6 -795,8 +796,8 @@@ func (runner *ContainerRunner) CreateCo runner.ContainerConfig.Env = append(runner.ContainerConfig.Env, k+"="+v) } + runner.ContainerConfig.Volumes = runner.Volumes + runner.HostConfig = dockercontainer.HostConfig{ Binds: runner.Binds, Cgroup: dockercontainer.CgroupSpec(runner.setCgroupParent), @@@ -1062,8 -1065,8 +1066,8 @@@ func (runner *ContainerRunner) CommitLo // point, but re-open crunch log with ArvClient in case there are any // other further (such as failing to write the log to Keep!) while // shutting down - runner.CrunchLog = NewThrottledLogger(&ArvLogWriter{runner.ArvClient, runner.Container.UUID, - "crunch-run", nil}) + runner.CrunchLog = NewThrottledLogger(&ArvLogWriter{ArvClient: runner.ArvClient, + UUID: runner.Container.UUID, loggingStream: "crunch-run", writeCloser: nil}) if runner.LogsPDH != nil { // If we have already assigned something to LogsPDH, @@@ -1150,8 -1153,7 +1154,8 @@@ func (runner *ContainerRunner) IsCancel // NewArvLogWriter creates an ArvLogWriter func (runner *ContainerRunner) NewArvLogWriter(name string) io.WriteCloser { - return &ArvLogWriter{runner.ArvClient, runner.Container.UUID, name, runner.LogCollection.Open(name + ".txt")} + return &ArvLogWriter{ArvClient: runner.ArvClient, UUID: runner.Container.UUID, loggingStream: name, + writeCloser: runner.LogCollection.Open(name + ".txt")} } // Run the full container lifecycle. @@@ -1291,9 -1293,6 +1295,9 @@@ func NewContainerRunner(api IArvadosCli cr.Container.UUID = containerUUID cr.CrunchLog = NewThrottledLogger(cr.NewLogWriter("crunch-run")) cr.CrunchLog.Immediate = log.New(os.Stderr, containerUUID+" ", 0) + + loadLogThrottleParams(api) + return cr }