From: Peter Amstutz Date: Wed, 25 Aug 2021 19:34:04 +0000 (-0400) Subject: Merge branch '17755-singularity-race' into main refs #17755 X-Git-Tag: 2.3.0~99 X-Git-Url: https://git.arvados.org/arvados.git/commitdiff_plain/d59079ca2a8e1afb3cf02bd9908c2cfe4c025d4f?hp=adc1cd0362bb9fd201b8c470eaf816ca5c212c72 Merge branch '17755-singularity-race' into main refs #17755 Arvados-DCO-1.1-Signed-off-by: Peter Amstutz --- diff --git a/tools/test-collection-create/test-collection-create.py b/tools/test-collection-create/test-collection-create.py new file mode 100644 index 0000000000..9a02745694 --- /dev/null +++ b/tools/test-collection-create/test-collection-create.py @@ -0,0 +1,443 @@ +#!/usr/bin/env python3 +# +# Copyright (C) The Arvados Authors. All rights reserved. +# +# SPDX-License-Identifier: CC-BY-SA-3.0 + +import argparse +import logging +import random +import string +import sys + +import arvados +import arvados.collection + +logger = logging.getLogger('arvados.test_collection_create') +logger.setLevel(logging.INFO) + +opts = argparse.ArgumentParser(add_help=False) +opts.add_argument('--min-files', type=int, default=30000, help=""" +Minimum number of files on each directory. Default: 30000. +""") +opts.add_argument('--max-files', type=int, default=30000, help=""" +Maximum number of files on each directory. Default: 30000. +""") +opts.add_argument('--min-depth', type=int, default=0, help=""" +Minimum depth for the created tree structure. Default: 0. +""") +opts.add_argument('--max-depth', type=int, default=0, help=""" +Maximum depth for the created tree structure. Default: 0. +""") +opts.add_argument('--min-subdirs', type=int, default=1, help=""" +Minimum number of subdirectories created at every depth level. Default: 1. +""") +opts.add_argument('--max-subdirs', type=int, default=10, help=""" +Maximum number of subdirectories created at every depth level. Default: 10. +""") +opts.add_argument('--debug', action='store_true', default=False, help=""" +Sets logging level to DEBUG. +""") + +arg_parser = argparse.ArgumentParser( + description='Create a collection with garbage data for testing purposes.', + parents=[opts]) + +adjectives = ['abandoned','able','absolute','adorable','adventurous','academic', + 'acceptable','acclaimed','accomplished','accurate','aching','acidic','acrobatic', + 'active','actual','adept','admirable','admired','adolescent','adorable','adored', + 'advanced','afraid','affectionate','aged','aggravating','aggressive','agile', + 'agitated','agonizing','agreeable','ajar','alarmed','alarming','alert','alienated', + 'alive','all','altruistic','amazing','ambitious','ample','amused','amusing','anchored', + 'ancient','angelic','angry','anguished','animated','annual','another','antique', + 'anxious','any','apprehensive','appropriate','apt','arctic','arid','aromatic','artistic', + 'ashamed','assured','astonishing','athletic','attached','attentive','attractive', + 'austere','authentic','authorized','automatic','avaricious','average','aware','awesome', + 'awful','awkward','babyish','bad','back','baggy','bare','barren','basic','beautiful', + 'belated','beloved','beneficial','better','best','bewitched','big','big-hearted', + 'biodegradable','bite-sized','bitter','black','black-and-white','bland','blank', + 'blaring','bleak','blind','blissful','blond','blue','blushing','bogus','boiling', + 'bold','bony','boring','bossy','both','bouncy','bountiful','bowed','brave','breakable', + 'brief','bright','brilliant','brisk','broken','bronze','brown','bruised','bubbly', + 'bulky','bumpy','buoyant','burdensome','burly','bustling','busy','buttery','buzzing', + 'calculating','calm','candid','canine','capital','carefree','careful','careless', + 'caring','cautious','cavernous','celebrated','charming','cheap','cheerful','cheery', + 'chief','chilly','chubby','circular','classic','clean','clear','clear-cut','clever', + 'close','closed','cloudy','clueless','clumsy','cluttered','coarse','cold','colorful', + 'colorless','colossal','comfortable','common','compassionate','competent','complete', + 'complex','complicated','composed','concerned','concrete','confused','conscious', + 'considerate','constant','content','conventional','cooked','cool','cooperative', + 'coordinated','corny','corrupt','costly','courageous','courteous','crafty','crazy', + 'creamy','creative','creepy','criminal','crisp','critical','crooked','crowded', + 'cruel','crushing','cuddly','cultivated','cultured','cumbersome','curly','curvy', + 'cute','cylindrical','damaged','damp','dangerous','dapper','daring','darling','dark', + 'dazzling','dead','deadly','deafening','dear','dearest','decent','decimal','decisive', + 'deep','defenseless','defensive','defiant','deficient','definite','definitive','delayed', + 'delectable','delicious','delightful','delirious','demanding','dense','dental', + 'dependable','dependent','descriptive','deserted','detailed','determined','devoted', + 'different','difficult','digital','diligent','dim','dimpled','dimwitted','direct', + 'disastrous','discrete','disfigured','disgusting','disloyal','dismal','distant', + 'downright','dreary','dirty','disguised','dishonest','dismal','distant','distinct', + 'distorted','dizzy','dopey','doting','double','downright','drab','drafty','dramatic', + 'dreary','droopy','dry','dual','dull','dutiful','each','eager','earnest','early', + 'easy','easy-going','ecstatic','edible','educated','elaborate','elastic','elated', + 'elderly','electric','elegant','elementary','elliptical','embarrassed','embellished', + 'eminent','emotional','empty','enchanted','enchanting','energetic','enlightened', + 'enormous','enraged','entire','envious','equal','equatorial','essential','esteemed', + 'ethical','euphoric','even','evergreen','everlasting','every','evil','exalted', + 'excellent','exemplary','exhausted','excitable','excited','exciting','exotic', + 'expensive','experienced','expert','extraneous','extroverted','extra-large','extra-small', + 'fabulous','failing','faint','fair','faithful','fake','false','familiar','famous', + 'fancy','fantastic','far','faraway','far-flung','far-off','fast','fat','fatal', + 'fatherly','favorable','favorite','fearful','fearless','feisty','feline','female', + 'feminine','few','fickle','filthy','fine','finished','firm','first','firsthand', + 'fitting','fixed','flaky','flamboyant','flashy','flat','flawed','flawless','flickering', + 'flimsy','flippant','flowery','fluffy','fluid','flustered','focused','fond','foolhardy', + 'foolish','forceful','forked','formal','forsaken','forthright','fortunate','fragrant', + 'frail','frank','frayed','free','French','fresh','frequent','friendly','frightened', + 'frightening','frigid','frilly','frizzy','frivolous','front','frosty','frozen', + 'frugal','fruitful','full','fumbling','functional','funny','fussy','fuzzy','gargantuan', + 'gaseous','general','generous','gentle','genuine','giant','giddy','gigantic','gifted', + 'giving','glamorous','glaring','glass','gleaming','gleeful','glistening','glittering', + 'gloomy','glorious','glossy','glum','golden','good','good-natured','gorgeous', + 'graceful','gracious','grand','grandiose','granular','grateful','grave','gray', + 'great','greedy','green','gregarious','grim','grimy','gripping','grizzled','gross', + 'grotesque','grouchy','grounded','growing','growling','grown','grubby','gruesome', + 'grumpy','guilty','gullible','gummy','hairy','half','handmade','handsome','handy', + 'happy','happy-go-lucky','hard','hard-to-find','harmful','harmless','harmonious', + 'harsh','hasty','hateful','haunting','healthy','heartfelt','hearty','heavenly', + 'heavy','hefty','helpful','helpless','hidden','hideous','high','high-level','hilarious', + 'hoarse','hollow','homely','honest','honorable','honored','hopeful','horrible', + 'hospitable','hot','huge','humble','humiliating','humming','humongous','hungry', + 'hurtful','husky','icky','icy','ideal','idealistic','identical','idle','idiotic', + 'idolized','ignorant','ill','illegal','ill-fated','ill-informed','illiterate', + 'illustrious','imaginary','imaginative','immaculate','immaterial','immediate', + 'immense','impassioned','impeccable','impartial','imperfect','imperturbable','impish', + 'impolite','important','impossible','impractical','impressionable','impressive', + 'improbable','impure','inborn','incomparable','incompatible','incomplete','inconsequential', + 'incredible','indelible','inexperienced','indolent','infamous','infantile','infatuated', + 'inferior','infinite','informal','innocent','insecure','insidious','insignificant', + 'insistent','instructive','insubstantial','intelligent','intent','intentional', + 'interesting','internal','international','intrepid','ironclad','irresponsible', + 'irritating','itchy','jaded','jagged','jam-packed','jaunty','jealous','jittery', + 'joint','jolly','jovial','joyful','joyous','jubilant','judicious','juicy','jumbo', + 'junior','jumpy','juvenile','kaleidoscopic','keen','key','kind','kindhearted','kindly', + 'klutzy','knobby','knotty','knowledgeable','knowing','known','kooky','kosher','lame', + 'lanky','large','last','lasting','late','lavish','lawful','lazy','leading','lean', + 'leafy','left','legal','legitimate','light','lighthearted','likable','likely','limited', + 'limp','limping','linear','lined','liquid','little','live','lively','livid','loathsome', + 'lone','lonely','long','long-term','loose','lopsided','lost','loud','lovable','lovely', + 'loving','low','loyal','lucky','lumbering','luminous','lumpy','lustrous','luxurious', + 'mad','made-up','magnificent','majestic','major','male','mammoth','married','marvelous', + 'masculine','massive','mature','meager','mealy','mean','measly','meaty','medical', + 'mediocre','medium','meek','mellow','melodic','memorable','menacing','merry','messy', + 'metallic','mild','milky','mindless','miniature','minor','minty','miserable','miserly', + 'misguided','misty','mixed','modern','modest','moist','monstrous','monthly','monumental', + 'moral','mortified','motherly','motionless','mountainous','muddy','muffled','multicolored', + 'mundane','murky','mushy','musty','muted','mysterious','naive','narrow','nasty','natural', + 'naughty','nautical','near','neat','necessary','needy','negative','neglected','negligible', + 'neighboring','nervous','new','next','nice','nifty','nimble','nippy','nocturnal','noisy', + 'nonstop','normal','notable','noted','noteworthy','novel','noxious','numb','nutritious', + 'nutty','obedient','obese','oblong','oily','oblong','obvious','occasional','odd', + 'oddball','offbeat','offensive','official','old','old-fashioned','only','open','optimal', + 'optimistic','opulent','orange','orderly','organic','ornate','ornery','ordinary', + 'original','other','our','outlying','outgoing','outlandish','outrageous','outstanding', + 'oval','overcooked','overdue','overjoyed','overlooked','palatable','pale','paltry', + 'parallel','parched','partial','passionate','past','pastel','peaceful','peppery', + 'perfect','perfumed','periodic','perky','personal','pertinent','pesky','pessimistic', + 'petty','phony','physical','piercing','pink','pitiful','plain','plaintive','plastic', + 'playful','pleasant','pleased','pleasing','plump','plush','polished','polite','political', + 'pointed','pointless','poised','poor','popular','portly','posh','positive','possible', + 'potable','powerful','powerless','practical','precious','present','prestigious', + 'pretty','precious','previous','pricey','prickly','primary','prime','pristine','private', + 'prize','probable','productive','profitable','profuse','proper','proud','prudent', + 'punctual','pungent','puny','pure','purple','pushy','putrid','puzzled','puzzling', + 'quaint','qualified','quarrelsome','quarterly','queasy','querulous','questionable', + 'quick','quick-witted','quiet','quintessential','quirky','quixotic','quizzical', + 'radiant','ragged','rapid','rare','rash','raw','recent','reckless','rectangular', + 'ready','real','realistic','reasonable','red','reflecting','regal','regular', + 'reliable','relieved','remarkable','remorseful','remote','repentant','required', + 'respectful','responsible','repulsive','revolving','rewarding','rich','rigid', + 'right','ringed','ripe','roasted','robust','rosy','rotating','rotten','rough', + 'round','rowdy','royal','rubbery','rundown','ruddy','rude','runny','rural','rusty', + 'sad','safe','salty','same','sandy','sane','sarcastic','sardonic','satisfied', + 'scaly','scarce','scared','scary','scented','scholarly','scientific','scornful', + 'scratchy','scrawny','second','secondary','second-hand','secret','self-assured', + 'self-reliant','selfish','sentimental','separate','serene','serious','serpentine', + 'several','severe','shabby','shadowy','shady','shallow','shameful','shameless', + 'sharp','shimmering','shiny','shocked','shocking','shoddy','short','short-term', + 'showy','shrill','shy','sick','silent','silky','silly','silver','similar','simple', + 'simplistic','sinful','single','sizzling','skeletal','skinny','sleepy','slight', + 'slim','slimy','slippery','slow','slushy','small','smart','smoggy','smooth','smug', + 'snappy','snarling','sneaky','sniveling','snoopy','sociable','soft','soggy','solid', + 'somber','some','spherical','sophisticated','sore','sorrowful','soulful','soupy', + 'sour','Spanish','sparkling','sparse','specific','spectacular','speedy','spicy', + 'spiffy','spirited','spiteful','splendid','spotless','spotted','spry','square', + 'squeaky','squiggly','stable','staid','stained','stale','standard','starchy','stark', + 'starry','steep','sticky','stiff','stimulating','stingy','stormy','straight','strange', + 'steel','strict','strident','striking','striped','strong','studious','stunning', + 'stupendous','stupid','sturdy','stylish','subdued','submissive','substantial','subtle', + 'suburban','sudden','sugary','sunny','super','superb','superficial','superior', + 'supportive','sure-footed','surprised','suspicious','svelte','sweaty','sweet','sweltering', + 'swift','sympathetic','tall','talkative','tame','tan','tangible','tart','tasty', + 'tattered','taut','tedious','teeming','tempting','tender','tense','tepid','terrible', + 'terrific','testy','thankful','that','these','thick','thin','third','thirsty','this', + 'thorough','thorny','those','thoughtful','threadbare','thrifty','thunderous','tidy', + 'tight','timely','tinted','tiny','tired','torn','total','tough','traumatic','treasured', + 'tremendous','tragic','trained','tremendous','triangular','tricky','trifling','trim', + 'trivial','troubled','true','trusting','trustworthy','trusty','truthful','tubby', + 'turbulent','twin','ugly','ultimate','unacceptable','unaware','uncomfortable', + 'uncommon','unconscious','understated','unequaled','uneven','unfinished','unfit', + 'unfolded','unfortunate','unhappy','unhealthy','uniform','unimportant','unique', + 'united','unkempt','unknown','unlawful','unlined','unlucky','unnatural','unpleasant', + 'unrealistic','unripe','unruly','unselfish','unsightly','unsteady','unsung','untidy', + 'untimely','untried','untrue','unused','unusual','unwelcome','unwieldy','unwilling', + 'unwitting','unwritten','upbeat','upright','upset','urban','usable','used','useful', + 'useless','utilized','utter','vacant','vague','vain','valid','valuable','vapid', + 'variable','vast','velvety','venerated','vengeful','verifiable','vibrant','vicious', + 'victorious','vigilant','vigorous','villainous','violet','violent','virtual', + 'virtuous','visible','vital','vivacious','vivid','voluminous','wan','warlike','warm', + 'warmhearted','warped','wary','wasteful','watchful','waterlogged','watery','wavy', + 'wealthy','weak','weary','webbed','wee','weekly','weepy','weighty','weird','welcome', + 'well-documented','well-groomed','well-informed','well-lit','well-made','well-off', + 'well-to-do','well-worn','wet','which','whimsical','whirlwind','whispered','white', + 'whole','whopping','wicked','wide','wide-eyed','wiggly','wild','willing','wilted', + 'winding','windy','winged','wiry','wise','witty','wobbly','woeful','wonderful', + 'wooden','woozy','wordy','worldly','worn','worried','worrisome','worse','worst', + 'worthless','worthwhile','worthy','wrathful','wretched','writhing','wrong','wry', + 'yawning','yearly','yellow','yellowish','young','youthful','yummy','zany','zealous', + 'zesty','zigzag'] +nouns = ['people','history','way','art','world','information','map','two','family', + 'government','health','system','computer','meat','year','thanks','music','person', + 'reading','method','data','food','understanding','theory','law','bird','literature', + 'problem','software','control','knowledge','power','ability','economics','love', + 'internet','television','science','library','nature','fact','product','idea', + 'temperature','investment','area','society','activity','story','industry','media', + 'thing','oven','community','definition','safety','quality','development','language', + 'management','player','variety','video','week','security','country','exam','movie', + 'organization','equipment','physics','analysis','policy','series','thought','basis', + 'boyfriend','direction','strategy','technology','army','camera','freedom','paper', + 'environment','child','instance','month','truth','marketing','university','writing', + 'article','department','difference','goal','news','audience','fishing','growth', + 'income','marriage','user','combination','failure','meaning','medicine','philosophy', + 'teacher','communication','night','chemistry','disease','disk','energy','nation', + 'road','role','soup','advertising','location','success','addition','apartment','education', + 'math','moment','painting','politics','attention','decision','event','property', + 'shopping','student','wood','competition','distribution','entertainment','office', + 'population','president','unit','category','cigarette','context','introduction', + 'opportunity','performance','driver','flight','length','magazine','newspaper', + 'relationship','teaching','cell','dealer','finding','lake','member','message','phone', + 'scene','appearance','association','concept','customer','death','discussion','housing', + 'inflation','insurance','mood','woman','advice','blood','effort','expression','importance', + 'opinion','payment','reality','responsibility','situation','skill','statement','wealth', + 'application','city','county','depth','estate','foundation','grandmother','heart', + 'perspective','photo','recipe','studio','topic','collection','depression','imagination', + 'passion','percentage','resource','setting','ad','agency','college','connection', + 'criticism','debt','description','memory','patience','secretary','solution','administration', + 'aspect','attitude','director','personality','psychology','recommendation','response', + 'selection','storage','version','alcohol','argument','complaint','contract','emphasis', + 'highway','loss','membership','possession','preparation','steak','union','agreement', + 'cancer','currency','employment','engineering','entry','interaction','mixture','preference', + 'region','republic','tradition','virus','actor','classroom','delivery','device', + 'difficulty','drama','election','engine','football','guidance','hotel','owner', + 'priority','protection','suggestion','tension','variation','anxiety','atmosphere', + 'awareness','bath','bread','candidate','climate','comparison','confusion','construction', + 'elevator','emotion','employee','employer','guest','height','leadership','mall','manager', + 'operation','recording','sample','transportation','charity','cousin','disaster','editor', + 'efficiency','excitement','extent','feedback','guitar','homework','leader','mom','outcome', + 'permission','presentation','promotion','reflection','refrigerator','resolution','revenue', + 'session','singer','tennis','basket','bonus','cabinet','childhood','church','clothes','coffee', + 'dinner','drawing','hair','hearing','initiative','judgment','lab','measurement','mode','mud', + 'orange','poetry','police','possibility','procedure','queen','ratio','relation','restaurant', + 'satisfaction','sector','signature','significance','song','tooth','town','vehicle','volume','wife', + 'accident','airport','appointment','arrival','assumption','baseball','chapter','committee', + 'conversation','database','enthusiasm','error','explanation','farmer','gate','girl','hall', + 'historian','hospital','injury','instruction','maintenance','manufacturer','meal','perception','pie', + 'poem','presence','proposal','reception','replacement','revolution','river','son','speech','tea', + 'village','warning','winner','worker','writer','assistance','breath','buyer','chest','chocolate', + 'conclusion','contribution','cookie','courage','dad','desk','drawer','establishment','examination', + 'garbage','grocery','honey','impression','improvement','independence','insect','inspection', + 'inspector','king','ladder','menu','penalty','piano','potato','profession','professor','quantity', + 'reaction','requirement','salad','sister','supermarket','tongue','weakness','wedding','affair', + 'ambition','analyst','apple','assignment','assistant','bathroom','bedroom','beer','birthday', + 'celebration','championship','cheek','client','consequence','departure','diamond','dirt','ear', + 'fortune','friendship','funeral','gene','girlfriend','hat','indication','intention','lady', + 'midnight','negotiation','obligation','passenger','pizza','platform','poet','pollution', + 'recognition','reputation','shirt','sir','speaker','stranger','surgery','sympathy','tale','throat', + 'trainer','uncle','youth','time','work','film','water','money','example','while','business','study', + 'game','life','form','air','day','place','number','part','field','fish','back','process','heat', + 'hand','experience','job','book','end','point','type','home','economy','value','body','market', + 'guide','interest','state','radio','course','company','price','size','card','list','mind','trade', + 'line','care','group','risk','word','fat','force','key','light','training','name','school','top', + 'amount','level','order','practice','research','sense','service','piece','web','boss','sport','fun', + 'house','page','term','test','answer','sound','focus','matter','kind','soil','board','oil','picture', + 'access','garden','range','rate','reason','future','site','demand','exercise','image','case','cause', + 'coast','action','age','bad','boat','record','result','section','building','mouse','cash','class', + 'nothing','period','plan','store','tax','side','subject','space','rule','stock','weather','chance', + 'figure','man','model','source','beginning','earth','program','chicken','design','feature','head', + 'material','purpose','question','rock','salt','act','birth','car','dog','object','scale','sun', + 'note','profit','rent','speed','style','war','bank','craft','half','inside','outside','standard', + 'bus','exchange','eye','fire','position','pressure','stress','advantage','benefit','box','frame', + 'issue','step','cycle','face','item','metal','paint','review','room','screen','structure','view', + 'account','ball','discipline','medium','share','balance','bit','black','bottom','choice','gift', + 'impact','machine','shape','tool','wind','address','average','career','culture','morning','pot', + 'sign','table','task','condition','contact','credit','egg','hope','ice','network','north','square', + 'attempt','date','effect','link','post','star','voice','capital','challenge','friend','self','shot', + 'brush','couple','debate','exit','front','function','lack','living','plant','plastic','spot', + 'summer','taste','theme','track','wing','brain','button','click','desire','foot','gas','influence', + 'notice','rain','wall','base','damage','distance','feeling','pair','savings','staff','sugar', + 'target','text','animal','author','budget','discount','file','ground','lesson','minute','officer', + 'phase','reference','register','sky','stage','stick','title','trouble','bowl','bridge','campaign', + 'character','club','edge','evidence','fan','letter','lock','maximum','novel','option','pack','park', + 'plenty','quarter','skin','sort','weight','baby','background','carry','dish','factor','fruit', + 'glass','joint','master','muscle','red','strength','traffic','trip','vegetable','appeal','chart', + 'gear','ideal','kitchen','land','log','mother','net','party','principle','relative','sale','season', + 'signal','spirit','street','tree','wave','belt','bench','commission','copy','drop','minimum','path', + 'progress','project','sea','south','status','stuff','ticket','tour','angle','blue','breakfast', + 'confidence','daughter','degree','doctor','dot','dream','duty','essay','father','fee','finance', + 'hour','juice','limit','luck','milk','mouth','peace','pipe','seat','stable','storm','substance', + 'team','trick','afternoon','bat','beach','blank','catch','chain','consideration','cream','crew', + 'detail','gold','interview','kid','mark','match','mission','pain','pleasure','score','screw','sex', + 'shop','shower','suit','tone','window','agent','band','block','bone','calendar','cap','coat', + 'contest','corner','court','cup','district','door','east','finger','garage','guarantee','hole', + 'hook','implement','layer','lecture','lie','manner','meeting','nose','parking','partner','profile', + 'respect','rice','routine','schedule','swimming','telephone','tip','winter','airline','bag','battle', + 'bed','bill','bother','cake','code','curve','designer','dimension','dress','ease','emergency', + 'evening','extension','farm','fight','gap','grade','holiday','horror','horse','host','husband', + 'loan','mistake','mountain','nail','noise','occasion','package','patient','pause','phrase','proof', + 'race','relief','sand','sentence','shoulder','smoke','stomach','string','tourist','towel','vacation', + 'west','wheel','wine','arm','aside','associate','bet','blow','border','branch','breast','brother', + 'buddy','bunch','chip','coach','cross','document','draft','dust','expert','floor','god','golf', + 'habit','iron','judge','knife','landscape','league','mail','mess','native','opening','parent', + 'pattern','pin','pool','pound','request','salary','shame','shelter','shoe','silver','tackle','tank', + 'trust','assist','bake','bar','bell','bike','blame','boy','brick','chair','closet','clue','collar', + 'comment','conference','devil','diet','fear','fuel','glove','jacket','lunch','monitor','mortgage', + 'nurse','pace','panic','peak','plane','reward','row','sandwich','shock','spite','spray','surprise', + 'till','transition','weekend','welcome','yard','alarm','bend','bicycle','bite','blind','bottle', + 'cable','candle','clerk','cloud','concert','counter','flower','grandfather','harm','knee','lawyer', + 'leather','load','mirror','neck','pension','plate','purple','ruin','ship','skirt','slice','snow', + 'specialist','stroke','switch','trash','tune','zone','anger','award','bid','bitter','boot','bug', + 'camp','candy','carpet','cat','champion','channel','clock','comfort','cow','crack','engineer', + 'entrance','fault','grass','guy','hell','highlight','incident','island','joke','jury','leg','lip', + 'mate','motor','nerve','passage','pen','pride','priest','prize','promise','resident','resort','ring', + 'roof','rope','sail','scheme','script','sock','station','toe','tower','truck','witness','a','you', + 'it','can','will','if','one','many','most','other','use','make','good','look','help','go','great', + 'being','few','might','still','public','read','keep','start','give','human','local','general','she', + 'specific','long','play','feel','high','tonight','put','common','set','change','simple','past','big', + 'possible','particular','today','major','personal','current','national','cut','natural','physical', + 'show','try','check','second','call','move','pay','let','increase','single','individual','turn', + 'ask','buy','guard','hold','main','offer','potential','professional','international','travel','cook', + 'alternative','following','special','working','whole','dance','excuse','cold','commercial','low', + 'purchase','deal','primary','worth','fall','necessary','positive','produce','search','present', + 'spend','talk','creative','tell','cost','drive','green','support','glad','remove','return','run', + 'complex','due','effective','middle','regular','reserve','independent','leave','original','reach', + 'rest','serve','watch','beautiful','charge','active','break','negative','safe','stay','visit', + 'visual','affect','cover','report','rise','walk','white','beyond','junior','pick','unique', + 'anything','classic','final','lift','mix','private','stop','teach','western','concern','familiar', + 'fly','official','broad','comfortable','gain','maybe','rich','save','stand','young','fail','heavy', + 'hello','lead','listen','valuable','worry','handle','leading','meet','release','sell','finish', + 'normal','press','ride','secret','spread','spring','tough','wait','brown','deep','display','flow', + 'hit','objective','shoot','touch','cancel','chemical','cry','dump','extreme','push','conflict','eat', + 'fill','formal','jump','kick','opposite','pass','pitch','remote','total','treat','vast','abuse', + 'beat','burn','deposit','print','raise','sleep','somewhere','advance','anywhere','consist','dark', + 'double','draw','equal','fix','hire','internal','join','kill','sensitive','tap','win','attack', + 'claim','constant','drag','drink','guess','minor','pull','raw','soft','solid','wear','weird', + 'wonder','annual','count','dead','doubt','feed','forever','impress','nobody','repeat','round','sing', + 'slide','strip','whereas','wish','combine','command','dig','divide','equivalent','hang','hunt', + 'initial','march','mention','smell','spiritual','survey','tie','adult','brief','crazy','escape', + 'gather','hate','prior','repair','rough','sad','scratch','sick','strike','employ','external','hurt', + 'illegal','laugh','lay','mobile','nasty','ordinary','respond','royal','senior','split','strain', + 'struggle','swim','train','upper','wash','yellow','convert','crash','dependent','fold','funny', + 'grab','hide','miss','permit','quote','recover','resolve','roll','sink','slip','spare','suspect', + 'sweet','swing','twist','upstairs','usual','abroad','brave','calm','concentrate','estimate','grand', + 'male','mine','prompt','quiet','refuse','regret','reveal','rush','shake','shift','shine','steal', + 'suck','surround','anybody','bear','brilliant','dare','dear','delay','drunk','female','hurry', + 'inevitable','invite','kiss','neat','pop','punch','quit','reply','representative','resist','rip', + 'rub','silly','smile','spell','stretch','stupid','tear','temporary','tomorrow','wake','wrap', + 'yesterday'] + +def get_random_name(with_ext=True): + return "{}_{}_{}{}".format( + random.choice(adjectives), + random.choice(nouns), + random.randint(0, 50000), + with_ext and '.txt' or '') + +def get_random_file(max_filesize): + file_start = random.randint(0, (max_filesize - 1025)) + file_size = random.randint(0, (max_filesize - file_start)) + file_name = get_random_name() + return "{}:{}:{}".format(file_start, file_size, file_name) + +def get_stream(name, max_filesize, data_loc, args): + files = [] + for _ in range(random.randint(args.min_files, args.max_files)): + files.append(get_random_file(max_filesize)) + stream = "{} {} {}".format(name, data_loc, ' '.join(files)) + return stream + +def create_substreams(depth, base_stream_name, max_filesize, data_loc, args, current_size=0): + current_stream = get_stream(base_stream_name, max_filesize, data_loc, args) + current_size += len(current_stream) + streams = [current_stream] + + if current_size >= (128 * 1024 * 1024): + logger.debug("Maximum manifest size reached -- finishing early at {}".format(base_stream_name)) + elif depth == 0: + logger.debug("Finished stream {}".format(base_stream_name)) + else: + for _ in range(random.randint(args.min_subdirs, args.max_subdirs)): + stream_name = base_stream_name+'/'+get_random_name(False) + substreams = create_substreams(depth-1, stream_name, max_filesize, + data_loc, args, current_size) + current_size += sum([len(x) for x in substreams]) + if current_size >= (128 * 1024 * 1024): + break + streams.extend(substreams) + return streams + +def parse_arguments(arguments): + args = arg_parser.parse_args(arguments) + if args.debug: + logger.setLevel(logging.DEBUG) + if args.max_files < args.min_files: + arg_parser.error("--min-files={} should be less or equal than max-files={}".format(args.min_files, args.max_files)) + if args.min_depth < 0: + arg_parser.error("--min-depth should be at least 0") + if args.max_depth < 0 or args.max_depth < args.min_depth: + arg_parser.error("--max-depth should be at >= 0 and >= min-depth={}".format(args.min_depth)) + if args.max_subdirs < args.min_subdirs: + arg_parser.error("--min-subdirs={} should be less or equal than max-subdirs={}".format(args.min_subdirs, args.max_subdirs)) + return args + +def main(arguments=None): + args = parse_arguments(arguments) + logger.info("Creating test collection with (min={}, max={}) files per directory and a tree depth of (min={}, max={}) and (min={}, max={}) subdirs in each depth level...".format(args.min_files, args.max_files, args.min_depth, args.max_depth, args.min_subdirs, args.max_subdirs)) + api = arvados.api('v1', timeout=5*60) + max_filesize = 1024*1024 + data_block = ''.join([random.choice(string.printable) for i in range(max_filesize)]) + data_loc = arvados.KeepClient(api).put(data_block) + streams = create_substreams(random.randint(args.min_depth, args.max_depth), + '.', max_filesize, data_loc, args) + manifest = '' + for s in streams: + if len(manifest)+len(s) > (1024*1024*128)-2: + logger.info("Skipping stream {} to avoid making a manifest bigger than 128MiB".format(s.split(' ')[0])) + break + manifest += s + '\n' + try: + coll_name = get_random_name(False) + coll = api.collections().create( + body={"collection": { + "name": coll_name, + "manifest_text": manifest + }, + }).execute() + except: + logger.info("ERROR creating collection with name '{}' and manifest:\n'{}...'\nSize: {}".format(coll_name, manifest[0:1024], len(manifest))) + raise + logger.info("Created collection {} - manifest size: {}".format(coll["uuid"], len(manifest))) + return 0 + +if __name__ == "__main__": + sys.exit(main()) \ No newline at end of file