|@like@, @ilike@|string|SQL pattern match, single character match is @_@ and wildcard is @%@, ilike is case-insensitive|@["properties.my_subproperty", "like", "d00220fb%"]@|
|@in@, @not in@|array of strings|Set membership|@["properties.my_subproperty", "in", ["fizz", "buzz"]]@|
|@exists@|boolean|Test if a subproperty is present or not (determined by operand).|@["properties.my_subproperty", "exists", true]@|
+|@contains@|string, number|Filter where subproperty has a value either by exact match or value is element of subproperty list.|@["foo", "contains", "bar"]@ will find both @{"foo": "bar"}@ and @{"foo": ["bar", "baz"]}@.|
Note that exclusion filters @!=@ and @not in@ will return records for which the property is not defined at all. To restrict filtering to records on which the subproperty is defined, combine with an @exists@ filter.
Subsystem: "dispatchcloud",
Name: "instances_total",
Help: "Number of cloud VMs.",
- }, []string{"category"})
+ }, []string{"category", "instance_type"})
reg.MustRegister(wp.mInstances)
wp.mInstancesPrice = prometheus.NewGaugeVec(prometheus.GaugeOpts{
Namespace: "arvados",
wp.mtx.RLock()
defer wp.mtx.RUnlock()
- instances := map[string]int64{}
+ type entKey struct {
+ cat string
+ instType string
+ }
+ instances := map[entKey]int64{}
price := map[string]float64{}
cpu := map[string]int64{}
mem := map[string]int64{}
default:
cat = "idle"
}
- instances[cat]++
+ instances[entKey{cat, wkr.instType.Name}]++
price[cat] += wkr.instType.Price
cpu[cat] += int64(wkr.instType.VCPUs)
mem[cat] += int64(wkr.instType.RAM)
running += int64(len(wkr.running) + len(wkr.starting))
}
for _, cat := range []string{"inuse", "hold", "booting", "unknown", "idle"} {
- wp.mInstances.WithLabelValues(cat).Set(float64(instances[cat]))
wp.mInstancesPrice.WithLabelValues(cat).Set(price[cat])
wp.mVCPUs.WithLabelValues(cat).Set(float64(cpu[cat]))
wp.mMemory.WithLabelValues(cat).Set(float64(mem[cat]))
+ // make sure to reset gauges for non-existing category/nodetype combinations
+ for _, it := range wp.instanceTypes {
+ if _, ok := instances[entKey{cat, it.Name}]; !ok {
+ wp.mInstances.WithLabelValues(cat, it.Name).Set(float64(0))
+ }
+ }
+ }
+ for k, v := range instances {
+ wp.mInstances.WithLabelValues(k.cat, k.instType).Set(float64(v))
}
wp.mContainersRunning.Set(float64(running))
}
func logResponse(w *responseTimer, req *http.Request, lgr *logrus.Entry) {
if tStart, ok := req.Context().Value(&requestTimeContextKey).(time.Time); ok {
tDone := time.Now()
+ writeTime := w.writeTime
+ if !w.wrote {
+ // Empty response body. Header was sent when
+ // handler exited.
+ writeTime = tDone
+ }
lgr = lgr.WithFields(logrus.Fields{
"timeTotal": stats.Duration(tDone.Sub(tStart)),
- "timeToStatus": stats.Duration(w.writeTime.Sub(tStart)),
- "timeWriteBody": stats.Duration(tDone.Sub(w.writeTime)),
+ "timeToStatus": stats.Duration(writeTime.Sub(tStart)),
+ "timeWriteBody": stats.Duration(tDone.Sub(writeTime)),
})
}
respCode := w.WroteStatus()
# format is YYYYMMDD, must be fixed with (needs to be linearly
# sortable), updated manually, may be used by clients to
# determine availability of API server features.
- revision: "20190926",
+ revision: "20200212",
source_version: AppVersion.hash,
sourceVersion: AppVersion.hash, # source_version should be deprecated in the future
packageVersion: AppVersion.package_version,
# +model_class+ subclass of ActiveRecord being filtered
#
# Output:
- # Hash with two keys:
+ # Hash with the following keys:
# :cond_out array of SQL fragments for each filter expression
# :param_out array of values for parameter substitution in cond_out
# :joins array of joins: either [] or ["JOIN containers ON ..."]
raise ArgumentError.new("Invalid operand '#{operand}' for '#{operator}' must be true or false")
end
param_out << proppath
+ when 'contains'
+ cond_out << "#{attr_table_name}.#{attr} @> ?::jsonb OR #{attr_table_name}.#{attr} @> ?::jsonb"
+ param_out << SafeJSON.dump({proppath => operand})
+ param_out << SafeJSON.dump({proppath => [operand]})
else
raise ArgumentError.new("Invalid operator for subproperty search '#{operator}'")
end
modified_at: 2015-02-13T17:22:54Z
updated_at: 2015-02-13T17:22:54Z
manifest_text: ". 37b51d194a7513e45b56f6524f2d51f2+3 0:3:bar\n"
- name: collection with prop1 1
+ name: collection with prop2 1
properties:
prop2: 1
modified_at: 2015-02-13T17:22:54Z
updated_at: 2015-02-13T17:22:54Z
manifest_text: ". 37b51d194a7513e45b56f6524f2d51f2+3 0:3:bar\n"
- name: collection with prop1 5
+ name: collection with prop2 5
properties:
prop2: 5
+collection_with_list_prop_odd:
+ uuid: zzzzz-4zz18-listpropertyodd
+ current_version_uuid: zzzzz-4zz18-listpropertyodd
+ portable_data_hash: fa7aeb5140e2848d39b416daeef4ffc5+45
+ owner_uuid: zzzzz-tpzed-xurymjxw79nv3jz
+ created_at: 2015-02-13T17:22:54Z
+ modified_by_client_uuid: zzzzz-ozdt8-brczlopd8u8d0jr
+ modified_by_user_uuid: zzzzz-tpzed-d9tiejq69daie8f
+ modified_at: 2015-02-13T17:22:54Z
+ updated_at: 2015-02-13T17:22:54Z
+ manifest_text: ". 37b51d194a7513e45b56f6524f2d51f2+3 0:3:bar\n"
+ name: collection with list property with odd values
+ properties:
+ listprop: [elem1, elem3, 5]
+
+collection_with_list_prop_even:
+ uuid: zzzzz-4zz18-listpropertyeven
+ current_version_uuid: zzzzz-4zz18-listpropertyeven
+ portable_data_hash: fa7aeb5140e2848d39b416daeef4ffc5+45
+ owner_uuid: zzzzz-tpzed-xurymjxw79nv3jz
+ created_at: 2015-02-13T17:22:54Z
+ modified_by_client_uuid: zzzzz-ozdt8-brczlopd8u8d0jr
+ modified_by_user_uuid: zzzzz-tpzed-d9tiejq69daie8f
+ modified_at: 2015-02-13T17:22:54Z
+ updated_at: 2015-02-13T17:22:54Z
+ manifest_text: ". 37b51d194a7513e45b56f6524f2d51f2+3 0:3:bar\n"
+ name: collection with list property with even values
+ properties:
+ listprop: [elem2, 4, elem6, ELEM8]
+
+collection_with_listprop_elem1:
+ uuid: zzzzz-4zz18-listpropelem1
+ current_version_uuid: zzzzz-4zz18-listpropelem1
+ portable_data_hash: fa7aeb5140e2848d39b416daeef4ffc5+45
+ owner_uuid: zzzzz-tpzed-xurymjxw79nv3jz
+ created_at: 2015-02-13T17:22:54Z
+ modified_by_client_uuid: zzzzz-ozdt8-brczlopd8u8d0jr
+ modified_by_user_uuid: zzzzz-tpzed-d9tiejq69daie8f
+ modified_at: 2015-02-13T17:22:54Z
+ updated_at: 2015-02-13T17:22:54Z
+ manifest_text: ". 37b51d194a7513e45b56f6524f2d51f2+3 0:3:bar\n"
+ name: collection with list property with string value
+ properties:
+ listprop: elem1
+
collection_with_uri_prop:
uuid: zzzzz-4zz18-withuripropval1
current_version_uuid: zzzzz-4zz18-withuripropval1
['prop2', '<=', 5, [:collection_with_prop2_1, :collection_with_prop2_5], []],
['prop2', '>=', 1, [:collection_with_prop2_1, :collection_with_prop2_5], []],
['<http://schema.org/example>', '=', "value1", [:collection_with_uri_prop], []],
+ ['listprop', 'contains', 'elem1', [:collection_with_list_prop_odd, :collection_with_listprop_elem1], [:collection_with_list_prop_even]],
+ ['listprop', '=', 'elem1', [:collection_with_listprop_elem1], [:collection_with_list_prop_odd]],
+ ['listprop', 'contains', 5, [:collection_with_list_prop_odd], [:collection_with_list_prop_even, :collection_with_listprop_elem1]],
+ ['listprop', 'contains', 'elem2', [:collection_with_list_prop_even], [:collection_with_list_prop_odd, :collection_with_listprop_elem1]],
+ ['listprop', 'contains', 'ELEM2', [], [:collection_with_list_prop_even]],
+ ['listprop', 'contains', 'elem8', [], [:collection_with_list_prop_even]],
+ ['listprop', 'contains', 4, [:collection_with_list_prop_even], [:collection_with_list_prop_odd, :collection_with_listprop_elem1]],
].each do |prop, op, opr, inc, ex|
test "jsonb filter properties.#{prop} #{op} #{opr})" do
@controller = Arvados::V1::CollectionsController.new
--- /dev/null
+# Copyright (C) The Arvados Authors. All rights reserved.
+#
+# SPDX-License-Identifier: AGPL-3.0
+
+case "$TARGET" in
+ centos*)
+ fpm_depends+=(mailcap)
+ ;;
+ debian* | ubuntu*)
+ fpm_depends+=(mime-support)
+ ;;
+esac
"git.arvados.org/arvados.git/lib/config"
"git.arvados.org/arvados.git/sdk/go/arvados"
+ "git.arvados.org/arvados.git/sdk/go/arvadosclient"
"git.arvados.org/arvados.git/sdk/go/arvadostest"
"git.arvados.org/arvados.git/sdk/go/auth"
"git.arvados.org/arvados.git/sdk/go/ctxlog"
s.Config = cfg
}
-func (s *UnitSuite) TestKeepClientBlockCache(c *check.C) {
- cfg := newConfig(s.Config)
- cfg.cluster.Collections.WebDAVCache.MaxBlockEntries = 42
- h := handler{Config: cfg}
- c.Check(keepclient.DefaultBlockCache.MaxBlocks, check.Not(check.Equals), cfg.cluster.Collections.WebDAVCache.MaxBlockEntries)
- u := mustParseURL("http://keep-web.example/c=" + arvadostest.FooCollection + "/t=" + arvadostest.ActiveToken + "/foo")
- req := &http.Request{
- Method: "GET",
- Host: u.Host,
- URL: u,
- RequestURI: u.RequestURI(),
- }
- resp := httptest.NewRecorder()
- h.ServeHTTP(resp, req)
- c.Check(resp.Code, check.Equals, http.StatusOK)
- c.Check(keepclient.DefaultBlockCache.MaxBlocks, check.Equals, cfg.cluster.Collections.WebDAVCache.MaxBlockEntries)
-}
-
func (s *UnitSuite) TestCORSPreflight(c *check.C) {
h := handler{Config: newConfig(s.Config)}
u := mustParseURL("http://keep-web.example/c=" + arvadostest.FooCollection + "/foo")
c.Check(resp.Body.String(), check.Matches, `{"health":"OK"}\n`)
}
+func (s *IntegrationSuite) TestFileContentType(c *check.C) {
+ s.testServer.Config.cluster.Services.WebDAVDownload.ExternalURL.Host = "download.example.com"
+
+ client := s.testServer.Config.Client
+ client.AuthToken = arvadostest.ActiveToken
+ arv, err := arvadosclient.New(&client)
+ c.Assert(err, check.Equals, nil)
+ kc, err := keepclient.MakeKeepClient(arv)
+ c.Assert(err, check.Equals, nil)
+
+ fs, err := (&arvados.Collection{}).FileSystem(&client, kc)
+ c.Assert(err, check.IsNil)
+
+ trials := []struct {
+ filename string
+ content string
+ contentType string
+ }{
+ {"picture.txt", "BMX bikes are small this year\n", "text/plain; charset=utf-8"},
+ {"picture.bmp", "BMX bikes are small this year\n", "image/x-ms-bmp"},
+ {"picture.jpg", "BMX bikes are small this year\n", "image/jpeg"},
+ {"picture1", "BMX bikes are small this year\n", "image/bmp"}, // content sniff; "BM" is the magic signature for .bmp
+ {"picture2", "Cars are small this year\n", "text/plain; charset=utf-8"}, // content sniff
+ }
+ for _, trial := range trials {
+ f, err := fs.OpenFile(trial.filename, os.O_CREATE|os.O_WRONLY, 0777)
+ c.Assert(err, check.IsNil)
+ _, err = f.Write([]byte(trial.content))
+ c.Assert(err, check.IsNil)
+ c.Assert(f.Close(), check.IsNil)
+ }
+ mtxt, err := fs.MarshalManifest(".")
+ c.Assert(err, check.IsNil)
+ var coll arvados.Collection
+ err = client.RequestAndDecode(&coll, "POST", "arvados/v1/collections", nil, map[string]interface{}{
+ "collection": map[string]string{
+ "manifest_text": mtxt,
+ },
+ })
+ c.Assert(err, check.IsNil)
+
+ for _, trial := range trials {
+ u, _ := url.Parse("http://download.example.com/by_id/" + coll.UUID + "/" + trial.filename)
+ req := &http.Request{
+ Method: "GET",
+ Host: u.Host,
+ URL: u,
+ RequestURI: u.RequestURI(),
+ Header: http.Header{
+ "Authorization": {"Bearer " + client.AuthToken},
+ },
+ }
+ resp := httptest.NewRecorder()
+ s.testServer.Handler.ServeHTTP(resp, req)
+ c.Check(resp.Code, check.Equals, http.StatusOK)
+ c.Check(resp.Header().Get("Content-Type"), check.Equals, trial.contentType)
+ c.Check(resp.Body.String(), check.Equals, trial.content)
+ }
+}
+
+func (s *IntegrationSuite) TestKeepClientBlockCache(c *check.C) {
+ s.testServer.Config.cluster.Collections.WebDAVCache.MaxBlockEntries = 42
+ c.Check(keepclient.DefaultBlockCache.MaxBlocks, check.Not(check.Equals), 42)
+ u := mustParseURL("http://keep-web.example/c=" + arvadostest.FooCollection + "/t=" + arvadostest.ActiveToken + "/foo")
+ req := &http.Request{
+ Method: "GET",
+ Host: u.Host,
+ URL: u,
+ RequestURI: u.RequestURI(),
+ }
+ resp := httptest.NewRecorder()
+ s.testServer.Handler.ServeHTTP(resp, req)
+ c.Check(resp.Code, check.Equals, http.StatusOK)
+ c.Check(keepclient.DefaultBlockCache.MaxBlocks, check.Equals, 42)
+}
+
func copyHeader(h http.Header) http.Header {
hc := http.Header{}
for k, v := range h {
import (
"flag"
"fmt"
+ "mime"
"os"
"git.arvados.org/arvados.git/lib/config"
log.Printf("keep-web %s started", version)
+ if ext := ".txt"; mime.TypeByExtension(ext) == "" {
+ log.Warnf("cannot look up MIME type for %q -- this probably means /etc/mime.types is missing -- clients will see incorrect content types", ext)
+ }
+
os.Setenv("ARVADOS_API_HOST", cfg.cluster.Services.Controller.ExternalURL.Host)
srv := &server{Config: cfg}
if err := srv.Start(); err != nil {
ENV GEM_PATH /var/lib/gems
ENV PATH $PATH:/var/lib/gems/bin
-ENV GOVERSION 1.12.7
+ENV GOVERSION 1.13.6
# Install golang binary
RUN curl -f http://storage.googleapis.com/golang/go${GOVERSION}.linux-amd64.tar.gz | \
# Can't use "yarn start", need to run the dev server script
# directly so that the TERM signal from "sv restart" gets to the
# right process.
+export VERSION=$(./version-at-commit.sh)
exec node node_modules/react-scripts-ts/scripts/start.js
projects / arvados.git / commitdiff