1 // A Volume is an interface representing a Keep back-end storage unit:
2 // for example, a single mounted disk, a RAID array, an Amazon S3 volume,
5 // A UnixVolume is a Volume that is backed by a locally mounted disk.
21 type Volume interface {
22 Read(loc string) ([]byte, error)
23 Write(loc string, block []byte) error
24 Index(prefix string) string
25 Status() *VolumeStatus
29 type UnixVolume struct {
30 root string // path to this volume
33 func (v *UnixVolume) String() string {
34 return fmt.Sprintf("[UnixVolume %s]", v.root)
37 // Read retrieves a block identified by the locator string "loc", and
38 // returns its contents as a byte slice.
40 // If the block could not be opened or read, Read returns a nil slice
41 // and the os.Error that was generated.
43 // If the block is present but its content hash does not match loc,
44 // Read returns the block and a CorruptError. It is the caller's
45 // responsibility to decide what (if anything) to do with the
46 // corrupted data block.
48 func (v *UnixVolume) Read(loc string) ([]byte, error) {
53 blockFilename := fmt.Sprintf("%s/%s/%s", v.root, loc[0:3], loc)
55 f, err = os.Open(blockFilename)
60 var buf = make([]byte, BLOCKSIZE)
61 nread, err = f.Read(buf)
63 log.Printf("%s: reading %s: %s\n", v, blockFilename, err)
68 return buf[:nread], nil
71 // Write stores a block of data identified by the locator string
72 // "loc". It returns nil on success. If the volume is full, it
73 // returns a FullError. If the write fails due to some other error,
74 // that error is returned.
76 func (v *UnixVolume) Write(loc string, block []byte) error {
80 blockDir := fmt.Sprintf("%s/%s", v.root, loc[0:3])
81 if err := os.MkdirAll(blockDir, 0755); err != nil {
82 log.Printf("%s: could not create directory %s: %s",
87 tmpfile, tmperr := ioutil.TempFile(blockDir, "tmp"+loc)
89 log.Printf("ioutil.TempFile(%s, tmp%s): %s", blockDir, loc, tmperr)
92 blockFilename := fmt.Sprintf("%s/%s", blockDir, loc)
94 if _, err := tmpfile.Write(block); err != nil {
95 log.Printf("%s: writing to %s: %s\n", v, blockFilename, err)
98 if err := tmpfile.Close(); err != nil {
99 log.Printf("closing %s: %s\n", tmpfile.Name(), err)
100 os.Remove(tmpfile.Name())
103 if err := os.Rename(tmpfile.Name(), blockFilename); err != nil {
104 log.Printf("rename %s %s: %s\n", tmpfile.Name(), blockFilename, err)
105 os.Remove(tmpfile.Name())
111 // Status returns a VolumeStatus struct describing the volume's
114 func (v *UnixVolume) Status() *VolumeStatus {
115 var fs syscall.Statfs_t
118 if fi, err := os.Stat(v.root); err == nil {
119 devnum = fi.Sys().(*syscall.Stat_t).Dev
121 log.Printf("%s: os.Stat: %s\n", v, err)
125 err := syscall.Statfs(v.root, &fs)
127 log.Printf("%s: statfs: %s\n", v, err)
130 // These calculations match the way df calculates disk usage:
131 // "free" space is measured by fs.Bavail, but "used" space
132 // uses fs.Blocks - fs.Bfree.
133 free := fs.Bavail * uint64(fs.Bsize)
134 used := (fs.Blocks - fs.Bfree) * uint64(fs.Bsize)
135 return &VolumeStatus{v.root, devnum, free, used}
138 // Index returns a list of blocks found on this volume which begin with
139 // the specified prefix. If the prefix is an empty string, Index returns
140 // a complete list of blocks.
142 // The return value is a multiline string (separated by
143 // newlines). Each line is in the format
145 // locator+size modification-time
149 // e4df392f86be161ca6ed3773a962b8f3+67108864 1388894303
150 // e4d41e6fd68460e0e3fc18cc746959d2+67108864 1377796043
151 // e4de7a2810f5554cd39b36d8ddb132ff+67108864 1388701136
153 func (v *UnixVolume) Index(prefix string) (output string) {
154 filepath.Walk(v.root,
155 func(path string, info os.FileInfo, err error) error {
156 // This WalkFunc inspects each path in the volume
157 // and prints an index line for all files that begin
160 log.Printf("IndexHandler: %s: walking to %s: %s",
164 locator := filepath.Base(path)
165 // Skip directories that do not match prefix.
166 // We know there is nothing interesting inside.
168 !strings.HasPrefix(locator, prefix) &&
169 !strings.HasPrefix(prefix, locator) {
170 return filepath.SkipDir
172 // Skip any file that is not apparently a locator, e.g. .meta files
173 if is_valid, err := IsValidLocator(locator); err != nil {
175 } else if !is_valid {
178 // Print filenames beginning with prefix
179 if !info.IsDir() && strings.HasPrefix(locator, prefix) {
180 output = output + fmt.Sprintf(
181 "%s+%d %d\n", locator, info.Size(), info.ModTime().Unix())
189 // IsFull returns true if the free space on the volume is less than
190 // MIN_FREE_KILOBYTES.
192 func (v *UnixVolume) IsFull() (isFull bool) {
193 fullSymlink := v.root + "/full"
195 // Check if the volume has been marked as full in the last hour.
196 if link, err := os.Readlink(fullSymlink); err == nil {
197 if ts, err := strconv.Atoi(link); err == nil {
198 fulltime := time.Unix(int64(ts), 0)
199 if time.Since(fulltime).Hours() < 1.0 {
205 if avail, err := v.FreeDiskSpace(); err == nil {
206 isFull = avail < MIN_FREE_KILOBYTES
208 log.Printf("%s: FreeDiskSpace: %s\n", v, err)
212 // If the volume is full, timestamp it.
214 now := fmt.Sprintf("%d", time.Now().Unix())
215 os.Symlink(now, fullSymlink)
220 // FreeDiskSpace returns the number of unused 1k blocks available on
223 func (v *UnixVolume) FreeDiskSpace() (free uint64, err error) {
224 var fs syscall.Statfs_t
225 err = syscall.Statfs(v.root, &fs)
227 // Statfs output is not guaranteed to measure free
228 // space in terms of 1K blocks.
229 free = fs.Bavail * uint64(fs.Bsize) / 1024