00001
00002
00003
00004
00005
00006
00007
00008
00009
00010
00011
00012
00013
00014
00015
00016
00017
00018 #include "FileSystemStore.h"
00019 #include "StorageConfig.h"
00020
00021 #include <sys/types.h>
00022 #include <sys/stat.h>
00023 #include <unistd.h>
00024 #include <dirent.h>
00025 #include <fcntl.h>
00026 #include <errno.h>
00027
00028 #include "../util/ExpandableBuffer.h"
00029 #include "../serialize/KeySerialize.h"
00030 #include "../serialize/TypeCollection.h"
00031 #include "../io/FileUtils.h"
00032 #include "../io/IO.h"
00033
00034
00035 namespace oasys {
00036
00037
00038 FileSystemStore::FileSystemStore(const char* logpath)
00039 : DurableStoreImpl("FileSystemStore", logpath),
00040 db_dir_("INVALID"),
00041 tables_dir_("INVALID"),
00042 default_perm_(S_IRUSR | S_IWUSR | S_IXUSR | S_IRGRP | S_IXGRP),
00043 fd_cache_(0)
00044 {}
00045
00046
00047 FileSystemStore::~FileSystemStore()
00048 {}
00049
00050
00051 int
00052 FileSystemStore::init(const StorageConfig& cfg)
00053 {
00054 if (cfg.dbdir_ == "") {
00055 return -1;
00056 }
00057
00058 if (cfg.dbname_ == "") {
00059 return -1;
00060 }
00061
00062 db_dir_ = cfg.dbdir_;
00063 FileUtils::abspath(&db_dir_);
00064
00065 tables_dir_ = db_dir_ + "/" + cfg.dbname_;
00066
00067 bool tidy = cfg.tidy_;
00068 bool init = cfg.init_;
00069
00070
00071
00072 if (tidy) {
00073 init = true;
00074 }
00075
00076 if (init && tidy)
00077 {
00078 if (check_database() == 0) {
00079 tidy_database();
00080 }
00081 int err = init_database();
00082 if (err != 0) {
00083 return -1;
00084 }
00085 }
00086 else if (init && ! tidy)
00087 {
00088 if (check_database() == -2) {
00089 int err = init_database();
00090 if (err != 0) {
00091 return -1;
00092 }
00093 }
00094 }
00095 else
00096 {
00097 if (check_database() != 0) {
00098 log_err("Database directory not found");
00099 return -1;
00100 }
00101 }
00102
00103 if (cfg.fs_fd_cache_size_ > 0) {
00104 fd_cache_ = new FdCache(logpath_, cfg.fs_fd_cache_size_);
00105 }
00106
00107 log_info("init() done");
00108 init_ = true;
00109
00110 return 0;
00111 }
00112
00113
00114 int
00115 FileSystemStore::get_table(DurableTableImpl** table,
00116 const std::string& name,
00117 int flags,
00118 PrototypeVector& prototypes)
00119 {
00120 (void)prototypes;
00121
00122 ASSERT(init_);
00123
00124 std::string dir_path = tables_dir_;
00125 dir_path.append("/");
00126 dir_path.append(name);
00127
00128 struct stat st;
00129 int err = stat(dir_path.c_str(), &st);
00130
00131
00132 if (err != 0 && errno == ENOENT) {
00133 if ( ! (flags & DS_CREATE)) {
00134 return DS_NOTFOUND;
00135 }
00136
00137 int err = mkdir(dir_path.c_str(), default_perm_);
00138 if (err != 0) {
00139 err = errno;
00140 log_err("Couldn't mkdir: %s", strerror(err));
00141 return DS_ERR;
00142 }
00143 } else if (err != 0) {
00144 return DS_ERR;
00145 } else if (err == 0 && (flags & DS_EXCL)) {
00146 return DS_EXISTS;
00147 }
00148
00149 FileSystemTable* table_ptr =
00150 new FileSystemTable(logpath_,
00151 name,
00152 dir_path,
00153 flags & DS_MULTITYPE,
00154 fd_cache_);
00155 ASSERT(table_ptr);
00156
00157 *table = table_ptr;
00158
00159 return 0;
00160 }
00161
00162
00163 int
00164 FileSystemStore::del_table(const std::string& name)
00165 {
00166
00167 ASSERT(init_);
00168
00169 std::string dir_path = tables_dir_;
00170 dir_path.append("/");
00171 dir_path.append(name);
00172
00173 FileUtils::rm_all_from_dir(dir_path.c_str());
00174
00175
00176 int err;
00177 err = rmdir(dir_path.c_str());
00178 if (err != 0) {
00179 log_warn("couldn't remove directory, %s", strerror(errno));
00180 return -1;
00181 }
00182
00183 return 0;
00184 }
00185
00186
00187 int
00188 FileSystemStore::get_table_names(StringVector* names)
00189 {
00190 names->clear();
00191
00192 DIR* dir = opendir(tables_dir_.c_str());
00193 if (dir == 0) {
00194 log_err("Can't get table names from directory");
00195 return DS_ERR;
00196 }
00197
00198 struct dirent* ent = readdir(dir);
00199 while (ent != 0) {
00200 names->push_back(ent->d_name);
00201 ent = readdir(dir);
00202 }
00203
00204 closedir(dir);
00205
00206 return 0;
00207 }
00208
00209
00210 std::string
00211 FileSystemStore::get_info() const
00212 {
00213 StringBuffer desc;
00214
00215 return "FileSystemStore";
00216 }
00217
00218
00219 int
00220 FileSystemStore::check_database()
00221 {
00222 DIR* dir = opendir(tables_dir_.c_str());
00223 if (dir == 0) {
00224 if (errno == ENOENT) {
00225 return -2;
00226 } else {
00227 return -1;
00228 }
00229 }
00230 closedir(dir);
00231
00232 return 0;
00233 }
00234
00235
00236 int
00237 FileSystemStore::init_database()
00238 {
00239 log_notice("init database (tables dir '%s'", tables_dir_.c_str());
00240
00241 int err = mkdir(db_dir_.c_str(), default_perm_);
00242 if (err != 0 && errno != EEXIST) {
00243 log_warn("init() failed: %s", strerror(errno));
00244 return -1;
00245 }
00246
00247 err = mkdir(tables_dir_.c_str(), default_perm_);
00248 if (err != 0 && errno != EEXIST) {
00249 log_warn("init() failed: %s", strerror(errno));
00250 return -1;
00251 }
00252
00253 return 0;
00254 }
00255
00256
00257 void
00258 FileSystemStore::tidy_database()
00259 {
00260 log_notice("Tidy() database, rm -rf %s", db_dir_.c_str());
00261
00262 char cmd[256];
00263 int cc = snprintf(cmd, 256, "rm -rf %s", db_dir_.c_str());
00264 ASSERT(cc < 256);
00265 system(cmd);
00266 }
00267
00268
00269 FileSystemTable::FileSystemTable(const char* logpath,
00270 const std::string& table_name,
00271 const std::string& path,
00272 bool multitype,
00273 FileSystemStore::FdCache* cache)
00274 : DurableTableImpl(table_name, multitype),
00275 Logger("FileSystemTable", "%s/%s", logpath, table_name.c_str()),
00276 path_(path),
00277 cache_(cache)
00278 {}
00279
00280
00281 FileSystemTable::~FileSystemTable()
00282 {}
00283
00284
00285 int
00286 FileSystemTable::get(const SerializableObject& key,
00287 SerializableObject* data)
00288 {
00289 ASSERTF(!multitype_, "single-type get called for multi-type table");
00290
00291 ScratchBuffer<u_char*, 4096> buf;
00292 int err = get_common(key, &buf);
00293 if (err != 0) {
00294 return err;
00295 }
00296
00297 Unmarshal um(Serialize::CONTEXT_LOCAL, buf.buf(), buf.len());
00298 err = um.action(data);
00299 if (err != 0) {
00300 return DS_ERR;
00301 }
00302
00303 return 0;
00304 }
00305
00306
00307 int
00308 FileSystemTable::get(const SerializableObject& key,
00309 SerializableObject** data,
00310 TypeCollection::Allocator_t allocator)
00311 {
00312 ASSERTF(multitype_, "multi-type get called for single-type table");
00313
00314 ScratchBuffer<u_char*, 4096> buf;
00315 int err = get_common(key, &buf);
00316 if (err != 0) {
00317 return err;
00318 }
00319
00320 Unmarshal um(Serialize::CONTEXT_LOCAL, buf.buf(), buf.len());
00321
00322 TypeCollection::TypeCode_t typecode;
00323 um.process("typecode", &typecode);
00324
00325 err = allocator(typecode, data);
00326 if (err != 0) {
00327 return DS_ERR;
00328 }
00329 err = um.action(*data);
00330 if (err != 0) {
00331 return DS_ERR;
00332 }
00333
00334 return 0;
00335 }
00336
00337
00338 int
00339 FileSystemTable::put(const SerializableObject& key,
00340 TypeCollection::TypeCode_t typecode,
00341 const SerializableObject* data,
00342 int flags)
00343 {
00344 ScratchBuffer<char*, 512> key_str;
00345 KeyMarshal s_key(&key_str, "-");
00346
00347 if (s_key.action(&key) != 0) {
00348 log_err("Can't get key");
00349 return DS_ERR;
00350 }
00351
00352 ScratchBuffer<u_char*, 4096> scratch;
00353 Marshal m(Serialize::CONTEXT_LOCAL, &scratch);
00354
00355 if (multitype_) {
00356 m.process("typecode", &typecode);
00357 }
00358 if (m.action(data) != 0) {
00359 log_warn("can't marshal data");
00360 return DS_ERR;
00361 }
00362
00363 std::string filename = path_ + "/" + key_str.buf();
00364 int data_elt_fd = -1;
00365 int open_flags = O_TRUNC | O_RDWR;
00366
00367 if (flags & DS_EXCL) {
00368 open_flags |= O_EXCL;
00369 }
00370
00371 if (flags & DS_CREATE) {
00372 open_flags |= O_CREAT;
00373 }
00374
00375 log_debug("opening file %s", filename.c_str());
00376
00377 if (cache_)
00378 {
00379 data_elt_fd = cache_->get_and_pin(filename);
00380 }
00381
00382 if (data_elt_fd == -1)
00383 {
00384 data_elt_fd = open(filename.c_str(), open_flags,
00385 S_IRUSR | S_IWUSR | S_IRGRP);
00386 if (data_elt_fd == -1)
00387 {
00388 if (errno == ENOENT)
00389 {
00390 ASSERT(! (flags & DS_CREATE));
00391 ASSERT(! (open_flags & O_CREAT));
00392 log_debug("file not found and DS_CREATE not specified");
00393 return DS_NOTFOUND;
00394 }
00395 else if (errno == EEXIST)
00396 {
00397 ASSERT(open_flags & O_EXCL);
00398 log_debug("file found and DS_EXCL specified");
00399 return DS_EXISTS;
00400 }
00401 else
00402 {
00403 log_warn("can't open %s: %s",
00404 filename.c_str(), strerror(errno));
00405 return DS_ERR;
00406 }
00407 }
00408
00409 if (cache_)
00410 {
00411 int old_fd = data_elt_fd;
00412 data_elt_fd = cache_->put_and_pin(filename, data_elt_fd);
00413 if (old_fd != data_elt_fd)
00414 {
00415 IO::close(old_fd);
00416 }
00417 }
00418 }
00419 else if (cache_ && (flags & DS_EXCL))
00420 {
00421 cache_->unpin(filename);
00422 return DS_EXISTS;
00423 }
00424
00425 log_debug("created file %s, fd = %d",
00426 filename.c_str(), data_elt_fd);
00427
00428 if (cache_)
00429 {
00430 int cc = IO::lseek(data_elt_fd, 0, SEEK_SET);
00431 ASSERT(cc == 0);
00432 }
00433
00434 int cc = IO::writeall(data_elt_fd,
00435 reinterpret_cast<char*>(scratch.buf()),
00436 scratch.len());
00437 if (cc != static_cast<int>(scratch.len()))
00438 {
00439 log_warn("put() - errors writing to file %s, %d: %s",
00440 filename.c_str(), cc, strerror(errno));
00441 if (cache_)
00442 {
00443 cache_->unpin(filename);
00444 }
00445 return DS_ERR;
00446 }
00447
00448 if (cache_)
00449 {
00450
00451 cache_->unpin(filename);
00452 }
00453 else
00454 {
00455 IO::close(data_elt_fd);
00456 }
00457
00458 return 0;
00459 }
00460
00461
00462 int
00463 FileSystemTable::del(const SerializableObject& key)
00464 {
00465 ScratchBuffer<char*, 512> key_str;
00466 KeyMarshal s_key(&key_str, "-");
00467
00468 if (s_key.action(&key) != 0) {
00469 log_err("Can't get key");
00470 return DS_ERR;
00471 }
00472
00473 std::string filename = path_ + "/" + key_str.buf();
00474
00475 if (cache_)
00476 {
00477 cache_->close(filename);
00478 }
00479
00480 int err = unlink(filename.c_str());
00481 if (err == -1)
00482 {
00483 if (errno == ENOENT) {
00484 return DS_NOTFOUND;
00485 }
00486
00487 log_warn("can't unlink file %s - %s", filename.c_str(),
00488 strerror(errno));
00489 return DS_ERR;
00490 }
00491
00492 return 0;
00493 }
00494
00495
00496 size_t
00497 FileSystemTable::size() const
00498 {
00499
00500 DIR* dir = opendir(path_.c_str());
00501 ASSERT(dir != 0);
00502
00503 size_t count;
00504 struct dirent* ent;
00505
00506 for (count = 0, ent = readdir(dir);
00507 ent != 0; ent = readdir(dir))
00508 {
00509 ASSERT(ent != 0);
00510 ++count;
00511 }
00512
00513 closedir(dir);
00514
00515
00516 log_debug("table size = %zu", count - 2);
00517
00518 return count - 2;
00519 }
00520
00521
00522 DurableIterator*
00523 FileSystemTable::itr()
00524 {
00525 return new FileSystemIterator(path_);
00526 }
00527
00528
00529 int
00530 FileSystemTable::get_common(const SerializableObject& key,
00531 ExpandableBuffer* buf)
00532 {
00533 ScratchBuffer<char*, 512> key_str;
00534 KeyMarshal serialize(&key_str, "-");
00535 if (serialize.action(&key) != 0) {
00536 log_err("Can't get key");
00537 return DS_ERR;
00538 }
00539
00540 std::string file_name(key_str.at(0));
00541 std::string file_path = path_ + "/" + file_name;
00542 log_debug("opening file %s", file_path.c_str());
00543
00544
00545 int fd = -1;
00546 if (cache_)
00547 {
00548 fd = cache_->get_and_pin(file_path);
00549 }
00550
00551 if (fd == -1) {
00552 fd = open(file_path.c_str(), O_RDWR);
00553 if (fd == -1)
00554 {
00555 log_debug("error opening file %s: %s",
00556 file_path.c_str(), strerror(errno));
00557 if (errno == ENOENT)
00558 {
00559 return DS_NOTFOUND;
00560 }
00561
00562 return DS_ERR;
00563 }
00564
00565 if (cache_)
00566 {
00567 int old_fd = fd;
00568 fd = cache_->put_and_pin(file_path, fd);
00569 if (old_fd != fd)
00570 {
00571 IO::close(old_fd);
00572 }
00573 }
00574 }
00575
00576 if (cache_)
00577 {
00578 int cc = IO::lseek(fd, 0, SEEK_SET);
00579 ASSERT(cc == 0);
00580 }
00581
00582 int cc;
00583 do {
00584 buf->reserve(buf->len() + 4096);
00585 cc = IO::read(fd, buf->end(), 4096, 0);
00586 ASSERTF(cc >= 0, "read failed %s", strerror(errno));
00587 buf->set_len(buf->len() + cc);
00588 } while (cc > 0);
00589
00590 if (cache_)
00591 {
00592 cache_->unpin(file_path);
00593 }
00594 else
00595 {
00596 IO::close(fd);
00597 }
00598
00599 return 0;
00600 }
00601
00602
00603 FileSystemIterator::FileSystemIterator(const std::string& path)
00604 : ent_(0)
00605 {
00606 dir_ = opendir(path.c_str());
00607 ASSERT(dir_ != 0);
00608 }
00609
00610
00611 FileSystemIterator::~FileSystemIterator()
00612 {
00613 closedir(dir_);
00614 }
00615
00616
00617 int
00618 FileSystemIterator::next()
00619 {
00620 skip_dots:
00621 ent_ = readdir(dir_);
00622
00623 if (ent_ != 0 && (strcmp(ent_->d_name, ".") == 0 ||
00624 strcmp(ent_->d_name, "..") == 0))
00625 {
00626 goto skip_dots;
00627 }
00628
00629 if (ent_ == 0)
00630 {
00631 if (errno == EBADF)
00632 {
00633 return DS_ERR;
00634 }
00635 else
00636 {
00637 return DS_NOTFOUND;
00638 }
00639 }
00640
00641 return 0;
00642 }
00643
00644
00645 int
00646 FileSystemIterator::get_key(SerializableObject* key)
00647 {
00648 ASSERT(ent_ != 0);
00649
00650 KeyUnmarshal um(ent_->d_name, strlen(ent_->d_name), "-");
00651 int err = um.action(key);
00652 if (err != 0) {
00653 return DS_ERR;
00654 }
00655
00656 return 0;
00657 }
00658
00659 }