1 // Testing tool for Keep services.
3 // keepexercise helps measure throughput and test reliability under
4 // various usage patterns.
6 // By default, it reads and writes blocks containing 2^26 NUL
7 // bytes. This generates network traffic without consuming much disk
10 // For a more realistic test, enable -vary-request. Warning: this will
11 // fill your storage volumes with random data if you leave it running,
12 // which can cost you money or leave you with too little room for
27 "git.curoverse.com/arvados.git/sdk/go/arvadosclient"
28 "git.curoverse.com/arvados.git/sdk/go/keepclient"
31 // Command line config knobs
33 BlockSize = flag.Int("block-size", keepclient.BLOCKSIZE, "bytes per read/write op")
34 ReadThreads = flag.Int("rthreads", 1, "number of concurrent readers")
35 WriteThreads = flag.Int("wthreads", 1, "number of concurrent writers")
36 VaryRequest = flag.Bool("vary-request", false, "vary the data for each request: consumes disk space, exercises write behavior")
37 VaryThread = flag.Bool("vary-thread", false, "use -wthreads different data blocks")
38 Replicas = flag.Int("replicas", 1, "replication level for writing")
39 StatsInterval = flag.Duration("stats-interval", time.Second, "time interval between IO stats reports, or 0 to disable")
40 ServiceURL = flag.String("url", "", "specify scheme://host of a single keep service to exercise (instead of using all advertised services like normal clients)")
41 ServiceUUID = flag.String("uuid", "", "specify UUID of a single advertised keep service to exercise")
47 arv, err := arvadosclient.MakeArvadosClient()
51 kc, err := keepclient.MakeKeepClient(arv)
55 kc.Want_replicas = *Replicas
57 transport := *(http.DefaultTransport.(*http.Transport))
58 transport.TLSClientConfig = arvadosclient.MakeTLSConfig(arv.ApiInsecure)
59 kc.HTTPClient = &http.Client{
60 Timeout: 10 * time.Minute,
61 Transport: &transport,
66 nextLocator := make(chan string, *ReadThreads+*WriteThreads)
68 go countBeans(nextLocator)
69 for i := 0; i < *WriteThreads; i++ {
70 nextBuf := make(chan []byte, 1)
71 go makeBufs(nextBuf, i)
72 go doWrites(kc, nextBuf, nextLocator)
74 for i := 0; i < *ReadThreads; i++ {
75 go doReads(kc, nextLocator)
80 // Send 1234 to bytesInChan when we receive 1234 bytes from keepstore.
81 var bytesInChan = make(chan uint64)
82 var bytesOutChan = make(chan uint64)
84 // Send struct{}{} to errorsChan when an error happens.
85 var errorsChan = make(chan struct{})
87 func countBeans(nextLocator chan string) {
89 var tickChan <-chan time.Time
90 if *StatsInterval > 0 {
91 tickChan = time.NewTicker(*StatsInterval).C
99 elapsed := time.Since(t0)
100 log.Printf("%v elapsed: read %v bytes (%.1f MiB/s), wrote %v bytes (%.1f MiB/s), errors %d",
102 bytesIn, (float64(bytesIn) / elapsed.Seconds() / 1048576),
103 bytesOut, (float64(bytesOut) / elapsed.Seconds() / 1048576),
106 case i := <-bytesInChan:
108 case o := <-bytesOutChan:
116 func makeBufs(nextBuf chan<- []byte, threadID int) {
117 buf := make([]byte, *BlockSize)
119 binary.PutVarint(buf, int64(threadID))
122 if randSize > *BlockSize {
123 randSize = *BlockSize
127 rnd := make([]byte, randSize)
128 if _, err := io.ReadFull(rand.Reader, rnd); err != nil {
131 buf = append(rnd, buf[randSize:]...)
137 func doWrites(kc *keepclient.KeepClient, nextBuf <-chan []byte, nextLocator chan<- string) {
138 for buf := range nextBuf {
139 locator, _, err := kc.PutB(buf)
142 errorsChan <- struct{}{}
145 bytesOutChan <- uint64(len(buf))
146 for cap(nextLocator) > len(nextLocator)+*WriteThreads {
147 // Give the readers something to do, unless
148 // they have lots queued up already.
149 nextLocator <- locator
154 func doReads(kc *keepclient.KeepClient, nextLocator <-chan string) {
155 for locator := range nextLocator {
156 rdr, size, url, err := kc.Get(locator)
159 errorsChan <- struct{}{}
162 n, err := io.Copy(ioutil.Discard, rdr)
164 if n != size || err != nil {
165 log.Printf("Got %d bytes (expected %d) from %s: %v", n, size, url, err)
166 errorsChan <- struct{}{}
168 // Note we don't count the bytes received in
169 // partial/corrupt responses: we are measuring
170 // throughput, not resource consumption.
172 bytesInChan <- uint64(n)
176 func overrideServices(kc *keepclient.KeepClient) {
177 roots := make(map[string]string)
178 if *ServiceURL != "" {
179 roots["zzzzz-bi6l4-000000000000000"] = *ServiceURL
180 } else if *ServiceUUID != "" {
181 for uuid, url := range kc.GatewayRoots() {
182 if uuid == *ServiceUUID {
188 log.Fatalf("Service %q was not in list advertised by API %+q", *ServiceUUID, kc.GatewayRoots())
193 kc.SetServiceRoots(roots, roots, roots)