--upload
If the build and test steps are successful, upload the python
packages to pypi and the gems to rubygems (default: false)
+--ruby <true|false>
+ Build ruby gems (default: true)
+--python <true|false>
+ Build python packages (default: true)
WORKSPACE=path Path to the Arvados source tree to build packages from
TARGET=
UPLOAD=0
+RUBY=1
+PYTHON=1
DEBUG=${ARVADOS_DEBUG:-0}
PARSEDOPTS=$(getopt --name "$0" --longoptions \
- help,debug,upload,target: \
+ help,debug,ruby:,python:,upload,target: \
-- "" "$@")
if [ $? -ne 0 ]; then
exit 1
--target)
TARGET="$2"; shift
;;
+ --ruby)
+ RUBY="$2"; shift
+ if [ "$RUBY" != "true" ] && [ "$RUBY" != "1" ]; then
+ RUBY=0
+ else
+ RUBY=1
+ fi
+ ;;
+ --python)
+ PYTHON="$2"; shift
+ if [ "$PYTHON" != "true" ] && [ "$PYTHON" != "1" ]; then
+ PYTHON=0
+ else
+ PYTHON=1
+ fi
+ ;;
--upload)
UPLOAD=1
;;
debug_echo "$0 is running from $RUN_BUILD_PACKAGES_PATH"
debug_echo "Workspace is $WORKSPACE"
+if [ $RUBY -eq 0 ] && [ $PYTHON -eq 0 ]; then
+ echo "Nothing to do!"
+ exit 0
+fi
+
if [[ -f /etc/profile.d/rvm.sh ]]; then
source /etc/profile.d/rvm.sh
GEM="rvm-exec default gem"
debug_echo "umask is" `umask`
-gem_wrapper arvados "$WORKSPACE/sdk/ruby"
-gem_wrapper arvados-cli "$WORKSPACE/sdk/cli"
-gem_wrapper arvados-login-sync "$WORKSPACE/services/login-sync"
-
GEM_BUILD_FAILURES=0
-if [ ${#failures[@]} -ne 0 ]; then
- GEM_BUILD_FAILURES=${#failures[@]}
+if [ $RUBY -eq 1 ]; then
+ debug_echo "Building Ruby gems"
+ gem_wrapper arvados "$WORKSPACE/sdk/ruby"
+ gem_wrapper arvados-cli "$WORKSPACE/sdk/cli"
+ gem_wrapper arvados-login-sync "$WORKSPACE/services/login-sync"
+ if [ ${#failures[@]} -ne 0 ]; then
+ GEM_BUILD_FAILURES=${#failures[@]}
+ fi
fi
-python_wrapper arvados-python-client "$WORKSPACE/sdk/python"
-python_wrapper arvados-pam "$WORKSPACE/sdk/pam"
-python_wrapper arvados-cwl-runner "$WORKSPACE/sdk/cwl"
-python_wrapper arvados_fuse "$WORKSPACE/services/fuse"
-python_wrapper arvados-node-manager "$WORKSPACE/services/nodemanager"
-
PYTHON_BUILD_FAILURES=0
-if [ $((${#failures[@]} - $GEM_BUILD_FAILURES)) -ne 0 ]; then
- PYTHON_BUILD_FAILURES=${#failures[@]} - $GEM_BUILD_FAILURES
+if [ $PYTHON -eq 1 ]; then
+ debug_echo "Building Python packages"
+ python_wrapper arvados-python-client "$WORKSPACE/sdk/python"
+ python_wrapper arvados-pam "$WORKSPACE/sdk/pam"
+ python_wrapper arvados-cwl-runner "$WORKSPACE/sdk/cwl"
+ python_wrapper arvados_fuse "$WORKSPACE/services/fuse"
+ python_wrapper arvados-node-manager "$WORKSPACE/services/nodemanager"
+
+ if [ $((${#failures[@]} - $GEM_BUILD_FAILURES)) -ne 0 ]; then
+ PYTHON_BUILD_FAILURES=$((${#failures[@]} - $GEM_BUILD_FAILURES))
+ fi
fi
-if [[ "$UPLOAD" != 0 ]]; then
+if [ $UPLOAD -ne 0 ]; then
+ echo "Uploading"
- if [[ $DEBUG > 0 ]]; then
+ if [ $DEBUG > 0 ]; then
EXTRA_UPLOAD_FLAGS=" --verbose"
else
EXTRA_UPLOAD_FLAGS=""
fi
- if [[ ! -e "$WORKSPACE/packages" ]]; then
+ if [ ! -e "$WORKSPACE/packages" ]; then
mkdir -p "$WORKSPACE/packages"
fi
- title "Start upload python packages"
- timer_reset
-
- if [ "$PYTHON_BUILD_FAILURES" -eq 0 ]; then
- /usr/local/arvados-dev/jenkins/run_upload_packages.py $EXTRA_UPLOAD_FLAGS --workspace $WORKSPACE python
- else
- echo "Skipping python packages upload, there were errors building the packages"
+ if [ $PYTHON -eq 1 ]; then
+ title "Start upload python packages"
+ timer_reset
+
+ if [ $PYTHON_BUILD_FAILURES -eq 0 ]; then
+ /usr/local/arvados-dev/jenkins/run_upload_packages.py $EXTRA_UPLOAD_FLAGS --workspace $WORKSPACE python
+ else
+ echo "Skipping python packages upload, there were errors building the packages"
+ fi
+ checkexit $? "upload python packages"
+ title "End of upload python packages (`timer`)"
fi
- checkexit $? "upload python packages"
- title "End of upload python packages (`timer`)"
- title "Start upload ruby gems"
- timer_reset
-
- if [ "$GEM_BUILD_FAILURES" -eq 0 ]; then
- /usr/local/arvados-dev/jenkins/run_upload_packages.py $EXTRA_UPLOAD_FLAGS --workspace $WORKSPACE gems
- else
- echo "Skipping ruby gem upload, there were errors building the packages"
+ if [ $RUBY -eq 1 ]; then
+ title "Start upload ruby gems"
+ timer_reset
+
+ if [ $GEM_BUILD_FAILURES -eq 0 ]; then
+ /usr/local/arvados-dev/jenkins/run_upload_packages.py $EXTRA_UPLOAD_FLAGS --workspace $WORKSPACE gems
+ else
+ echo "Skipping ruby gem upload, there were errors building the packages"
+ fi
+ checkexit $? "upload ruby gems"
+ title "End of upload ruby gems (`timer`)"
fi
- checkexit $? "upload ruby gems"
- title "End of upload ruby gems (`timer`)"
-
fi
exit_cleanly
ClientSecret: "zzzzzzzzzzzzzzzzzzzzzzzz"
</pre>
+Check the OpenIDConnect section in the "default config file":{{site.baseurl}}/admin/config.html for more details and configuration options.
+
h2(#ldap). LDAP
With this configuration, authentication uses an external LDAP service like OpenLDAP or Active Directory.
ClientID: ""
ClientSecret: ""
+ # OpenID claim field containing the user's email
+ # address. Normally "email"; see
+ # https://openid.net/specs/openid-connect-core-1_0.html#StandardClaims
+ EmailClaim: "email"
+
+ # OpenID claim field containing the email verification
+ # flag. Normally "email_verified". To accept every returned
+ # email address without checking a "verified" field at all,
+ # use the empty string "".
+ EmailVerifiedClaim: "email_verified"
+
+ # OpenID claim field containing the user's preferred
+ # username. If empty, use the mailbox part of the user's email
+ # address.
+ UsernameClaim: ""
+
PAM:
# (Experimental) Use PAM to authenticate users.
Enable: false
"Login.OpenIDConnect.ClientSecret": false,
"Login.OpenIDConnect.Enable": true,
"Login.OpenIDConnect.Issuer": false,
+ "Login.OpenIDConnect.EmailClaim": false,
+ "Login.OpenIDConnect.EmailVerifiedClaim": false,
+ "Login.OpenIDConnect.UsernameClaim": false,
"Login.PAM": true,
"Login.PAM.DefaultEmailDomain": false,
"Login.PAM.Enable": true,
ClientID: ""
ClientSecret: ""
+ # OpenID claim field containing the user's email
+ # address. Normally "email"; see
+ # https://openid.net/specs/openid-connect-core-1_0.html#StandardClaims
+ EmailClaim: "email"
+
+ # OpenID claim field containing the email verification
+ # flag. Normally "email_verified". To accept every returned
+ # email address without checking a "verified" field at all,
+ # use the empty string "".
+ EmailVerifiedClaim: "email_verified"
+
+ # OpenID claim field containing the user's preferred
+ # username. If empty, use the mailbox part of the user's email
+ # address.
+ UsernameClaim: ""
+
PAM:
# (Experimental) Use PAM to authenticate users.
Enable: false
ClientID: cluster.Login.Google.ClientID,
ClientSecret: cluster.Login.Google.ClientSecret,
UseGooglePeopleAPI: cluster.Login.Google.AlternateEmailAddresses,
+ EmailClaim: "email",
+ EmailVerifiedClaim: "email_verified",
}
case !wantGoogle && wantOpenIDConnect && !wantSSO && !wantPAM && !wantLDAP:
return &oidcLoginController{
- Cluster: cluster,
- RailsProxy: railsProxy,
- Issuer: cluster.Login.OpenIDConnect.Issuer,
- ClientID: cluster.Login.OpenIDConnect.ClientID,
- ClientSecret: cluster.Login.OpenIDConnect.ClientSecret,
+ Cluster: cluster,
+ RailsProxy: railsProxy,
+ Issuer: cluster.Login.OpenIDConnect.Issuer,
+ ClientID: cluster.Login.OpenIDConnect.ClientID,
+ ClientSecret: cluster.Login.OpenIDConnect.ClientSecret,
+ EmailClaim: cluster.Login.OpenIDConnect.EmailClaim,
+ EmailVerifiedClaim: cluster.Login.OpenIDConnect.EmailVerifiedClaim,
+ UsernameClaim: cluster.Login.OpenIDConnect.UsernameClaim,
}
case !wantGoogle && !wantOpenIDConnect && wantSSO && !wantPAM && !wantLDAP:
return &ssoLoginController{railsProxy}
Issuer string // OIDC issuer URL, e.g., "https://accounts.google.com"
ClientID string
ClientSecret string
- UseGooglePeopleAPI bool // Use Google People API to look up alternate email addresses
+ UseGooglePeopleAPI bool // Use Google People API to look up alternate email addresses
+ EmailClaim string // OpenID claim to use as email address; typically "email"
+ EmailVerifiedClaim string // If non-empty, ensure claim value is true before accepting EmailClaim; typically "email_verified"
+ UsernameClaim string // If non-empty, use as preferred username
// override Google People API base URL for testing purposes
// (normally empty, set by google pkg to
var ret rpc.UserSessionAuthInfo
defer ctxlog.FromContext(ctx).WithField("ret", &ret).Debug("getAuthInfo returned")
- var claims struct {
- Name string `json:"name"`
- Email string `json:"email"`
- Verified bool `json:"email_verified"`
- }
+ var claims map[string]interface{}
if err := idToken.Claims(&claims); err != nil {
return nil, fmt.Errorf("error extracting claims from ID token: %s", err)
- } else if claims.Verified {
+ } else if verified, _ := claims[ctrl.EmailVerifiedClaim].(bool); verified || ctrl.EmailVerifiedClaim == "" {
// Fall back to this info if the People API call
// (below) doesn't return a primary && verified email.
- if names := strings.Fields(strings.TrimSpace(claims.Name)); len(names) > 1 {
+ name, _ := claims["name"].(string)
+ if names := strings.Fields(strings.TrimSpace(name)); len(names) > 1 {
ret.FirstName = strings.Join(names[0:len(names)-1], " ")
ret.LastName = names[len(names)-1]
} else {
ret.FirstName = names[0]
}
- ret.Email = claims.Email
+ ret.Email, _ = claims[ctrl.EmailClaim].(string)
+ }
+
+ if ctrl.UsernameClaim != "" {
+ ret.Username, _ = claims[ctrl.UsernameClaim].(string)
}
if !ctrl.UseGooglePeopleAPI {
if ret.Email == "" {
- return nil, fmt.Errorf("cannot log in with unverified email address %q", claims.Email)
+ return nil, fmt.Errorf("cannot log in with unverified email address %q", claims[ctrl.EmailClaim])
}
return &ret, nil
}
return nil, errors.New("cannot log in without a verified email address")
}
for ae := range altEmails {
- if ae != ret.Email {
- ret.AlternateEmails = append(ret.AlternateEmails, ae)
- if i := strings.Index(ae, "@"); i > 0 && strings.ToLower(ae[i+1:]) == strings.ToLower(ctrl.Cluster.Users.PreferDomainForUsername) {
+ if ae == ret.Email {
+ continue
+ }
+ ret.AlternateEmails = append(ret.AlternateEmails, ae)
+ if ret.Username == "" {
+ i := strings.Index(ae, "@")
+ if i > 0 && strings.ToLower(ae[i+1:]) == strings.ToLower(ctrl.Cluster.Users.PreferDomainForUsername) {
ret.Username = strings.SplitN(ae[:i], "+", 2)[0]
}
}
"email": s.authEmail,
"email_verified": s.authEmailVerified,
"name": s.authName,
+ "alt_verified": true, // for custom claim tests
+ "alt_email": "alt_email@example.com", // for custom claim tests
+ "alt_username": "desired-username", // for custom claim tests
})
json.NewEncoder(w).Encode(struct {
AccessToken string `json:"access_token"`
c.Check(resp.RedirectLocation, check.Equals, "")
}
-func (s *OIDCLoginSuite) TestOIDCLogin_Success(c *check.C) {
+func (s *OIDCLoginSuite) TestGenericOIDCLogin(c *check.C) {
s.cluster.Login.Google.Enable = false
s.cluster.Login.OpenIDConnect.Enable = true
json.Unmarshal([]byte(fmt.Sprintf("%q", s.fakeIssuer.URL)), &s.cluster.Login.OpenIDConnect.Issuer)
s.cluster.Login.OpenIDConnect.ClientSecret = "oidc#client#secret"
s.validClientID = "oidc#client#id"
s.validClientSecret = "oidc#client#secret"
- s.localdb = NewConn(s.cluster)
- state := s.startLogin(c)
- resp, err := s.localdb.Login(context.Background(), arvados.LoginOptions{
- Code: s.validCode,
- State: state,
- })
- c.Assert(err, check.IsNil)
- c.Check(resp.HTML.String(), check.Equals, "")
- target, err := url.Parse(resp.RedirectLocation)
- c.Assert(err, check.IsNil)
- token := target.Query().Get("api_token")
- c.Check(token, check.Matches, `v2/zzzzz-gj3su-.{15}/.{32,50}`)
+ for _, trial := range []struct {
+ expectEmail string // "" if failure expected
+ setup func()
+ }{
+ {
+ expectEmail: "user@oidc.example.com",
+ setup: func() {
+ c.Log("=== succeed because email_verified is false but not required")
+ s.authEmail = "user@oidc.example.com"
+ s.authEmailVerified = false
+ s.cluster.Login.OpenIDConnect.EmailClaim = "email"
+ s.cluster.Login.OpenIDConnect.EmailVerifiedClaim = ""
+ s.cluster.Login.OpenIDConnect.UsernameClaim = ""
+ },
+ },
+ {
+ expectEmail: "",
+ setup: func() {
+ c.Log("=== fail because email_verified is false and required")
+ s.authEmail = "user@oidc.example.com"
+ s.authEmailVerified = false
+ s.cluster.Login.OpenIDConnect.EmailClaim = "email"
+ s.cluster.Login.OpenIDConnect.EmailVerifiedClaim = "email_verified"
+ s.cluster.Login.OpenIDConnect.UsernameClaim = ""
+ },
+ },
+ {
+ expectEmail: "user@oidc.example.com",
+ setup: func() {
+ c.Log("=== succeed because email_verified is false but config uses custom 'verified' claim")
+ s.authEmail = "user@oidc.example.com"
+ s.authEmailVerified = false
+ s.cluster.Login.OpenIDConnect.EmailClaim = "email"
+ s.cluster.Login.OpenIDConnect.EmailVerifiedClaim = "alt_verified"
+ s.cluster.Login.OpenIDConnect.UsernameClaim = ""
+ },
+ },
+ {
+ expectEmail: "alt_email@example.com",
+ setup: func() {
+ c.Log("=== succeed with custom 'email' and 'email_verified' claims")
+ s.authEmail = "bad@wrong.example.com"
+ s.authEmailVerified = false
+ s.cluster.Login.OpenIDConnect.EmailClaim = "alt_email"
+ s.cluster.Login.OpenIDConnect.EmailVerifiedClaim = "alt_verified"
+ s.cluster.Login.OpenIDConnect.UsernameClaim = "alt_username"
+ },
+ },
+ } {
+ trial.setup()
+ if s.railsSpy != nil {
+ s.railsSpy.Close()
+ }
+ s.railsSpy = arvadostest.NewProxy(c, s.cluster.Services.RailsAPI)
+ s.localdb = NewConn(s.cluster)
+ *s.localdb.railsProxy = *rpc.NewConn(s.cluster.ClusterID, s.railsSpy.URL, true, rpc.PassthroughTokenProvider)
+
+ state := s.startLogin(c)
+ resp, err := s.localdb.Login(context.Background(), arvados.LoginOptions{
+ Code: s.validCode,
+ State: state,
+ })
+ c.Assert(err, check.IsNil)
+ if trial.expectEmail == "" {
+ c.Check(resp.HTML.String(), check.Matches, `(?ms).*Login error.*`)
+ c.Check(resp.RedirectLocation, check.Equals, "")
+ continue
+ }
+ c.Check(resp.HTML.String(), check.Equals, "")
+ target, err := url.Parse(resp.RedirectLocation)
+ c.Assert(err, check.IsNil)
+ token := target.Query().Get("api_token")
+ c.Check(token, check.Matches, `v2/zzzzz-gj3su-.{15}/.{32,50}`)
+ authinfo := getCallbackAuthInfo(c, s.railsSpy)
+ c.Check(authinfo.Email, check.Equals, trial.expectEmail)
+
+ switch s.cluster.Login.OpenIDConnect.UsernameClaim {
+ case "alt_username":
+ c.Check(authinfo.Username, check.Equals, "desired-username")
+ case "":
+ c.Check(authinfo.Username, check.Equals, "")
+ default:
+ c.Fail() // bad test case
+ }
+ }
}
func (s *OIDCLoginSuite) TestGoogleLogin_Success(c *check.C) {
import cwltool.process
import cwltool.argparser
from cwltool.process import shortname, UnsupportedRequirement, use_custom_schema
-from cwltool.pathmapper import adjustFileObjs, adjustDirObjs, get_listing
+from cwltool.utils import adjustFileObjs, adjustDirObjs, get_listing
import arvados
import arvados.config
cwltool.command_line_tool.ACCEPTLIST_RE = cwltool.command_line_tool.ACCEPTLIST_EN_RELAXED_RE
res10 = pkg_resources.resource_stream(__name__, 'arv-cwl-schema-v1.0.yml')
res11 = pkg_resources.resource_stream(__name__, 'arv-cwl-schema-v1.1.yml')
- customschema10 = res10.read()
- customschema11 = res11.read()
+ customschema10 = res10.read().decode('utf-8')
+ customschema11 = res11.read().decode('utf-8')
use_custom_schema("v1.0", "http://arvados.org/cwl", customschema10)
use_custom_schema("v1.1.0-dev1", "http://arvados.org/cwl", customschema11)
use_custom_schema("v1.1", "http://arvados.org/cwl", customschema11)
from ._version import __version__
from cwltool.process import shortname, UnsupportedRequirement, use_custom_schema
-from cwltool.pathmapper import adjustFileObjs, adjustDirObjs, get_listing, visit_class
+from cwltool.utils import adjustFileObjs, adjustDirObjs, get_listing, visit_class
from cwltool.command_line_tool import compute_checksums
from cwltool.load_tool import load_tool
# file to determine what version of cwltool and schema-salad to
# build.
install_requires=[
- 'cwltool==3.0.20200324120055',
- 'schema-salad==5.0.20200302192450',
+ 'cwltool==3.0.20200530110633',
+ 'schema-salad==6.0.20200601095207',
'arvados-python-client{}'.format(pysdk_dep),
'setuptools',
'ciso8601 >= 2.0.0'
AlternateEmailAddresses bool
}
OpenIDConnect struct {
- Enable bool
- Issuer string
- ClientID string
- ClientSecret string
+ Enable bool
+ Issuer string
+ ClientID string
+ ClientSecret string
+ EmailClaim string
+ EmailVerifiedClaim string
+ UsernameClaim string
}
PAM struct {
Enable bool
"log"
"net/http"
"os"
+ "os/signal"
+ "syscall"
"time"
"git.arvados.org/arvados.git/sdk/go/arvadosclient"
ServiceURL = flag.String("url", "", "specify scheme://host of a single keep service to exercise (instead of using all advertised services like normal clients)")
ServiceUUID = flag.String("uuid", "", "specify UUID of a single advertised keep service to exercise")
getVersion = flag.Bool("version", false, "Print version information and exit.")
+ RunTime = flag.Duration("run-time", 0, "time to run (e.g. 60s), or 0 to run indefinitely (default)")
)
func main() {
os.Exit(0)
}
- log.Printf("keep-exercise %s started", version)
+ stderr := log.New(os.Stderr, "", log.LstdFlags)
arv, err := arvadosclient.MakeArvadosClient()
if err != nil {
- log.Fatal(err)
+ stderr.Fatal(err)
}
kc, err := keepclient.MakeKeepClient(arv)
if err != nil {
- log.Fatal(err)
+ stderr.Fatal(err)
}
kc.Want_replicas = *Replicas
Transport: &transport,
}
- overrideServices(kc)
+ overrideServices(kc, stderr)
nextLocator := make(chan string, *ReadThreads+*WriteThreads)
- go countBeans(nextLocator)
+ go countBeans(nextLocator, stderr)
for i := 0; i < *WriteThreads; i++ {
nextBuf := make(chan []byte, 1)
- go makeBufs(nextBuf, i)
- go doWrites(kc, nextBuf, nextLocator)
+ go makeBufs(nextBuf, i, stderr)
+ go doWrites(kc, nextBuf, nextLocator, stderr)
}
for i := 0; i < *ReadThreads; i++ {
- go doReads(kc, nextLocator)
+ go doReads(kc, nextLocator, stderr)
}
<-make(chan struct{})
}
// Send struct{}{} to errorsChan when an error happens.
var errorsChan = make(chan struct{})
-func countBeans(nextLocator chan string) {
+func countBeans(nextLocator chan string, stderr *log.Logger) {
t0 := time.Now()
var tickChan <-chan time.Time
+ var endChan <-chan time.Time
+ c := make(chan os.Signal)
+ signal.Notify(c, os.Interrupt, syscall.SIGTERM)
if *StatsInterval > 0 {
tickChan = time.NewTicker(*StatsInterval).C
}
+ if *RunTime > 0 {
+ endChan = time.NewTicker(*RunTime).C
+ }
var bytesIn uint64
var bytesOut uint64
var errors uint64
+ var rateIn, rateOut float64
+ var maxRateIn, maxRateOut float64
+ var abort, printCsv bool
+ csv := log.New(os.Stdout, "", 0)
+ csv.Println("Timestamp,Elapsed,Read (bytes),Avg Read Speed (MiB/s),Peak Read Speed (MiB/s),Written (bytes),Avg Write Speed (MiB/s),Peak Write Speed (MiB/s),Errors,ReadThreads,WriteThreads,VaryRequest,VaryThread,BlockSize,Replicas,StatsInterval,ServiceURL,ServiceUUID,RunTime")
for {
select {
case <-tickChan:
- elapsed := time.Since(t0)
- log.Printf("%v elapsed: read %v bytes (%.1f MiB/s), wrote %v bytes (%.1f MiB/s), errors %d",
- elapsed,
- bytesIn, (float64(bytesIn) / elapsed.Seconds() / 1048576),
- bytesOut, (float64(bytesOut) / elapsed.Seconds() / 1048576),
- errors,
- )
+ printCsv = true
+ case <-endChan:
+ printCsv = true
+ abort = true
+ case <-c:
+ printCsv = true
+ abort = true
+ fmt.Print("\r") // Suppress the ^C print
case i := <-bytesInChan:
bytesIn += i
case o := <-bytesOutChan:
case <-errorsChan:
errors++
}
+ if printCsv {
+ elapsed := time.Since(t0)
+ rateIn = float64(bytesIn) / elapsed.Seconds() / 1048576
+ if rateIn > maxRateIn {
+ maxRateIn = rateIn
+ }
+ rateOut = float64(bytesOut) / elapsed.Seconds() / 1048576
+ if rateOut > maxRateOut {
+ maxRateOut = rateOut
+ }
+ csv.Printf("%v,%v,%v,%.1f,%.1f,%v,%.1f,%.1f,%d,%d,%d,%t,%t,%d,%d,%s,%s,%s,%s",
+ time.Now().Format("2006-01-02 15:04:05"),
+ elapsed,
+ bytesIn, rateIn, maxRateIn,
+ bytesOut, rateOut, maxRateOut,
+ errors,
+ *ReadThreads,
+ *WriteThreads,
+ *VaryRequest,
+ *VaryThread,
+ *BlockSize,
+ *Replicas,
+ *StatsInterval,
+ *ServiceURL,
+ *ServiceUUID,
+ *RunTime,
+ )
+ printCsv = false
+ }
+ if abort {
+ os.Exit(0)
+ }
}
}
-func makeBufs(nextBuf chan<- []byte, threadID int) {
+func makeBufs(nextBuf chan<- []byte, threadID int, stderr *log.Logger) {
buf := make([]byte, *BlockSize)
if *VaryThread {
binary.PutVarint(buf, int64(threadID))
if *VaryRequest {
rnd := make([]byte, randSize)
if _, err := io.ReadFull(rand.Reader, rnd); err != nil {
- log.Fatal(err)
+ stderr.Fatal(err)
}
buf = append(rnd, buf[randSize:]...)
}
}
}
-func doWrites(kc *keepclient.KeepClient, nextBuf <-chan []byte, nextLocator chan<- string) {
+func doWrites(kc *keepclient.KeepClient, nextBuf <-chan []byte, nextLocator chan<- string, stderr *log.Logger) {
for buf := range nextBuf {
locator, _, err := kc.PutB(buf)
if err != nil {
- log.Print(err)
+ stderr.Print(err)
errorsChan <- struct{}{}
continue
}
}
}
-func doReads(kc *keepclient.KeepClient, nextLocator <-chan string) {
+func doReads(kc *keepclient.KeepClient, nextLocator <-chan string, stderr *log.Logger) {
for locator := range nextLocator {
rdr, size, url, err := kc.Get(locator)
if err != nil {
- log.Print(err)
+ stderr.Print(err)
errorsChan <- struct{}{}
continue
}
n, err := io.Copy(ioutil.Discard, rdr)
rdr.Close()
if n != size || err != nil {
- log.Printf("Got %d bytes (expected %d) from %s: %v", n, size, url, err)
+ stderr.Printf("Got %d bytes (expected %d) from %s: %v", n, size, url, err)
errorsChan <- struct{}{}
continue
// Note we don't count the bytes received in
}
}
-func overrideServices(kc *keepclient.KeepClient) {
+func overrideServices(kc *keepclient.KeepClient, stderr *log.Logger) {
roots := make(map[string]string)
if *ServiceURL != "" {
roots["zzzzz-bi6l4-000000000000000"] = *ServiceURL
}
}
if len(roots) == 0 {
- log.Fatalf("Service %q was not in list advertised by API %+q", *ServiceUUID, kc.GatewayRoots())
+ stderr.Fatalf("Service %q was not in list advertised by API %+q", *ServiceUUID, kc.GatewayRoots())
}
} else {
return
projects / arvados.git / commitdiff