1 module FileBucket; 2 3 private import tango.io.device.File; 4 5 private import tango.core.Exception; 6 7 /****************************************************************************** 8 9 FileBucket implements a simple mechanism to store and recover a 10 large quantity of data for the duration of the hosting process. 11 It is intended to act as a local-cache for a remote data-source, 12 or as a spillover area for large in-memory cache instances. 13 14 Note that any and all stored data is rendered invalid the moment 15 a FileBucket object is garbage-collected. 16 17 The implementation follows a fixed-capacity record scheme, where 18 content can be rewritten in-place until said capacity is reached. 19 At such time, the altered content is moved to a larger capacity 20 record at end-of-file, and a hole remains at the prior location. 21 These holes are not collected, since the lifespan of a FileBucket 22 is limited to that of the host process. 23 24 All index keys must be unique. Writing to the FileBucket with an 25 existing key will overwrite any previous content. What follows 26 is a contrived example: 27 28 Example: 29 --- 30 const(char)[] text = "this is a test"; 31 32 auto bucket = new FileBucket ("bucket.bin", FileBucket.FileBucket.HalfK); 33 34 // insert some data, and retrieve it again 35 bucket.put ("a key", text); 36 char[] b = cast(char[]) bucket.get ("a key"); 37 38 assert (b == text); 39 bucket.close; 40 --- 41 42 ******************************************************************************/ 43 44 class FileBucket 45 { 46 /********************************************************************** 47 48 Define the capacity (block-size) of each record 49 50 **********************************************************************/ 51 52 struct BlockSize 53 { 54 int capacity; 55 } 56 57 // basic capacity for each record 58 private const(char)[] path; 59 60 // basic capacity for each record 61 private BlockSize block; 62 63 // where content is stored 64 private File file; 65 66 // pointers to file records 67 private Record[char[]] map; 68 69 // current file size 70 private long fileSize; 71 72 // current file usage 73 private long waterLine; 74 75 // supported block sizes 76 public static enum BlockSize EighthK = {128-1}, 77 HalfK = {512-1}, 78 OneK = {1024*1-1}, 79 TwoK = {1024*2-1}, 80 FourK = {1024*4-1}, 81 EightK = {1024*8-1}, 82 SixteenK = {1024*16-1}, 83 ThirtyTwoK = {1024*32-1}, 84 SixtyFourK = {1024*64-1}; 85 86 87 /********************************************************************** 88 89 Construct a FileBucket with the provided path, record-size, 90 and inital record count. The latter causes records to be 91 pre-allocated, saving a certain amount of growth activity. 92 Selecting a record size that roughly matches the serialized 93 content will limit 'thrashing'. 94 95 **********************************************************************/ 96 97 this (const(char)[] path, BlockSize block, uint initialRecords = 100) 98 { 99 this.path = path; 100 this.block = block; 101 102 // open a storage file 103 file = new File (path, File.ReadWriteCreate); 104 105 // set initial file size (can be zero) 106 fileSize = initialRecords * block.capacity; 107 file.seek (fileSize); 108 file.truncate (); 109 } 110 111 /********************************************************************** 112 113 Return the block-size in use for this FileBucket 114 115 **********************************************************************/ 116 117 int getBufferSize () const 118 { 119 return block.capacity+1; 120 } 121 122 /********************************************************************** 123 124 Return where the FileBucket is located 125 126 **********************************************************************/ 127 128 const(char)[] getFilePath () const 129 { 130 return path; 131 } 132 133 /********************************************************************** 134 135 Return the currently populated size of this FileBucket 136 137 **********************************************************************/ 138 139 long length () const 140 { 141 synchronized(this) 142 { 143 return waterLine; 144 } 145 } 146 147 /********************************************************************** 148 149 Return the serialized data for the provided key. Returns 150 null if the key was not found. 151 152 **********************************************************************/ 153 154 void[] get (const(char)[] key) 155 { 156 synchronized(this) 157 { 158 Record r = null; 159 160 if (key in map) 161 { 162 r = map [key]; 163 return r.read (this); 164 } 165 return null; 166 } 167 } 168 169 /********************************************************************** 170 171 Remove the provided key from this FileBucket. 172 173 **********************************************************************/ 174 175 void remove (const(char)[] key) 176 { 177 synchronized(this) 178 { 179 map.remove(key); 180 } 181 } 182 183 /********************************************************************** 184 185 Write a serialized block of data, and associate it with 186 the provided key. All keys must be unique, and it is the 187 responsibility of the programmer to ensure this. Reusing 188 an existing key will overwrite previous data. 189 190 Note that data is allowed to grow within the occupied 191 bucket until it becomes larger than the allocated space. 192 When this happens, the data is moved to a larger bucket 193 at the file tail. 194 195 **********************************************************************/ 196 197 void put (const(char)[] key, const(char)[] data) 198 { 199 synchronized(this) 200 { 201 Record* r = key in map; 202 203 if (r is null) 204 { 205 auto rr = new Record; 206 map [key] = rr; 207 r = &rr; 208 } 209 r.write (this, data, block); 210 } 211 } 212 213 /********************************************************************** 214 215 Close this FileBucket -- all content is lost. 216 217 **********************************************************************/ 218 219 void close () 220 { 221 synchronized(this) 222 { 223 if (file) 224 { 225 file.detach(); 226 file = null; 227 map = null; 228 } 229 } 230 } 231 232 /********************************************************************** 233 234 Each Record takes up a number of 'pages' within the file. 235 The size of these pages is determined by the BlockSize 236 provided during FileBucket construction. Additional space 237 at the end of each block is potentially wasted, but enables 238 content to grow in size without creating a myriad of holes. 239 240 **********************************************************************/ 241 242 private static class Record 243 { 244 private long offset; 245 private long length, 246 capacity = -1; 247 248 /************************************************************** 249 250 **************************************************************/ 251 252 private static void eof (FileBucket bucket) 253 { 254 throw new IOException ("Unexpected EOF in FileBucket '"~bucket.path.idup~"'"); 255 } 256 257 /************************************************************** 258 259 This should be protected from thread-contention at 260 a higher level. 261 262 **************************************************************/ 263 264 void[] read (FileBucket bucket) 265 { 266 void[] data = new ubyte [length]; 267 268 bucket.file.seek (offset); 269 if (bucket.file.read (data) != length) 270 eof (bucket); 271 272 return data; 273 } 274 275 /************************************************************** 276 277 This should be protected from thread-contention at 278 a higher level. 279 280 **************************************************************/ 281 282 void write (FileBucket bucket, const(void)[] data, BlockSize block) 283 { 284 length = data.length; 285 286 // create new slot if we exceed capacity 287 if (length > capacity) 288 createBucket (bucket, length, block); 289 290 // locate to start of content 291 bucket.file.seek (offset); 292 293 // write content 294 if (bucket.file.write (data) != length) 295 eof (bucket); 296 } 297 298 /************************************************************** 299 300 **************************************************************/ 301 302 void createBucket (FileBucket bucket, long bytes, BlockSize block) 303 { 304 offset = bucket.waterLine; 305 capacity = (bytes + block.capacity) & ~block.capacity; 306 307 bucket.waterLine += capacity; 308 if (bucket.waterLine > bucket.fileSize) 309 { 310 // grow the filesize 311 bucket.fileSize = bucket.waterLine * 2; 312 313 // expand the physical file size 314 bucket.file.seek (bucket.fileSize); 315 bucket.file.truncate (); 316 } 317 } 318 } 319 } 320 321