/** * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with this * work for additional information regarding copyright ownership. The ASF * licenses this file to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance with the License. * You may obtain a copy of the License at *

* http://www.apache.org/licenses/LICENSE-2.0 *

* Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS,WITHOUT * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the * License for the specific language governing permissions and limitations under * the License. */ #define FUSE_USE_VERSION 31 #ifdef HAVE_CONFIG_H #include #endif #ifdef linux /* For pread()/pwrite()/utimensat() */ #define _XOPEN_SOURCE 700 #endif #include #include #include #include #include #include #include #include #include #include #include #ifdef __FreeBSD__ #include #include #endif #include #ifdef HAVE_SETXATTR #include #endif #include "failure_injector_fs.h" #include "failure_injector.h" #include "run_grpc_service.h" #include #include #include #include #include #include using NoiseInjector::FailureInjector; using NoiseInjector::RunGrpcService; using namespace std; using namespace NoiseInjector::FileSystem; #define LOGFILE "/var/log/noise_injector_log.txt" struct fuse_operations FailureInjectorFs::mFuseOperationsVec; FailureInjector *FailureInjectorFs::mFailureInjector = NULL; RunGrpcService *FailureInjectorFs::mGrpcSever = NULL; FailureInjectorFs::FailureInjectorFs(FailureInjector *injector) { mFailureInjector = injector; mGrpcSever = new RunGrpcService(mFailureInjector); } int FailureInjectorFs::mknod_wrapper( int dirfd, const char *path, const char *link, int mode, dev_t rdev) { int res; if (S_ISREG(mode)) { res = openat(dirfd, path, O_CREAT | O_EXCL | O_WRONLY, mode); if (res >= 0) res = close(res); } else if (S_ISDIR(mode)) { res = mkdirat(dirfd, path, mode); } else if (S_ISLNK(mode) && link != NULL) { res = symlinkat(link, dirfd, path); } else if (S_ISFIFO(mode)) { res = mkfifoat(dirfd, path, mode); } else { res = mknodat(dirfd, path, mode, rdev); } return res; } void *FailureInjectorFs::fifs_init( struct fuse_conn_info *conn, struct fuse_config *cfg) { extern void RunServer(); (void) conn; cfg->use_ino = 1; std::freopen(LOGFILE, "w", stdout); /* * Pick up changes from lower filesystem right away. This is * also necessary for better hardlink support. When the kernel * calls the unlink() handler, it does not know the inode of * the to-be-removed entry and can therefore not invalidate * the cache of the associated inode - resulting in an * incorrect st_nlink value being reported for any remaining * hardlinks to this inode. */ cfg->entry_timeout = 0; cfg->attr_timeout = 0; cfg->negative_timeout = 0; /* Create a separate GRPC server thread to accept failure injection. */ std::thread th(&RunGrpcService::RunServer, mGrpcSever); th.detach(); return NULL; } int FailureInjectorFs::fifs_getattr( const char *path, struct stat *stbuf, struct fuse_file_info *fi) { (void) fi; int res; int injected_error = 0; if (CheckForInjectedError(path,"GETATTR", &injected_error)) { cout << "Returning Injected error " << injected_error << "\n"; return -injected_error; } res = lstat(path, stbuf); if (res == -1) return -errno; return 0; } int FailureInjectorFs::fifs_access( const char *path, int mask) { int res; int injected_error = 0; if (CheckForInjectedError(path,"ACCESS", &injected_error)) { cout << "Returning Injected error " << injected_error << "\n"; return -injected_error; } res = access(path, mask); if (res == -1) return -errno; return 0; } int FailureInjectorFs::fifs_readlink( const char *path, char *buf, size_t size) { int res; int injected_error = 0; if (CheckForInjectedError(path,"READLINK", &injected_error)) { cout << "Returning Injected error " << injected_error << "\n"; return -injected_error; } res = readlink(path, buf, size - 1); if (res == -1) return -errno; buf[res] = '\0'; return 0; } int FailureInjectorFs::fifs_readdir( const char *path, void *buf, fuse_fill_dir_t filler, off_t offset, struct fuse_file_info *fi, enum fuse_readdir_flags flags) { DIR *dp; struct dirent *de; int injected_error = 0; (void) offset; (void) fi; (void) flags; if (CheckForInjectedError(path,"READDIR", &injected_error)) { cout << "Returning Injected error " << injected_error << "\n"; return -injected_error; } dp = opendir(path); if (dp == NULL) return -errno; while ((de = readdir(dp)) != NULL) { struct stat st; memset(&st, 0, sizeof(st)); st.st_ino = de->d_ino; st.st_mode = de->d_type << 12; if (filler(buf, de->d_name, &st, 0, (fuse_fill_dir_flags)0)) break; } closedir(dp); return 0; } int FailureInjectorFs::fifs_mknod( const char *path, mode_t mode, dev_t rdev) { int res; int injected_error = 0; if (CheckForInjectedError(path,"MKNOD", &injected_error)) { cout << "Returning Injected error " << injected_error << "\n"; return -injected_error; } res = mknod_wrapper(AT_FDCWD, path, NULL, mode, rdev); if (res == -1) return -errno; return 0; } int FailureInjectorFs::fifs_mkdir( const char *path, mode_t mode) { int res; int injected_error = 0; if (CheckForInjectedError(path,"MKDIR", &injected_error)) { cout << "Returning Injected error " << injected_error << "\n"; return -injected_error; } res = mkdir(path, mode); if (res == -1) return -errno; return 0; } int FailureInjectorFs::fifs_unlink(const char *path) { int res; int injected_error = 0; if (CheckForInjectedError(path,"UNLINK", &injected_error)) { cout << "Returning Injected error " << injected_error << "\n"; return -injected_error; } res = unlink(path); if (res == -1) return -errno; return 0; } int FailureInjectorFs::fifs_rmdir(const char *path) { int res; int injected_error = 0; if (CheckForInjectedError(path,"RMDIR", &injected_error)) { cout << "Returning Injected error " << injected_error << "\n"; return -injected_error; } res = rmdir(path); if (res == -1) return -errno; return 0; } int FailureInjectorFs::fifs_symlink( const char *from, const char *to) { int res; int injected_error = 0; if (CheckForInjectedError(from, "SYMLINK", &injected_error)) { cout << "Returning Injected error " << injected_error << "\n"; return -injected_error; } res = symlink(from, to); if (res == -1) return -errno; return 0; } int FailureInjectorFs::fifs_rename( const char *from, const char *to, unsigned int flags) { int res; int injected_error = 0; if (CheckForInjectedError(from, "RENAME", &injected_error)) { cout << "Returning Injected error " << injected_error << "\n"; return -injected_error; } if (flags) return -EINVAL; res = rename(from, to); if (res == -1) return -errno; return 0; } int FailureInjectorFs::fifs_link( const char *from, const char *to) { int res; int injected_error = 0; if (CheckForInjectedError(from,"LINK", &injected_error)) { cout << "Returning Injected error " << injected_error << "\n"; return -injected_error; } res = link(from, to); if (res == -1) return -errno; return 0; } int FailureInjectorFs::fifs_chmod( const char *path, mode_t mode, struct fuse_file_info *fi) { (void) fi; int res; int injected_error = 0; if (CheckForInjectedError(path,"CHMOD", &injected_error)) { cout << "Returning Injected error " << injected_error << "\n"; return -injected_error; } res = chmod(path, mode); if (res == -1) return -errno; return 0; } int FailureInjectorFs::fifs_chown( const char *path, uid_t uid, gid_t gid, struct fuse_file_info *fi) { (void) fi; int res; int injected_error = 0; if (CheckForInjectedError(path,"CHOWN", &injected_error)) { cout << "Returning Injected error " << injected_error << "\n"; return -injected_error; } res = lchown(path, uid, gid); if (res == -1) return -errno; return 0; } int FailureInjectorFs::fifs_truncate( const char *path, off_t size, struct fuse_file_info *fi) { int res; int injected_error = 0; if (CheckForInjectedError(path,"TRUNCATE", &injected_error)) { cout << "Returning Injected error " << injected_error << "\n"; return -injected_error; } if (fi != NULL) res = ftruncate(fi->fh, size); else res = truncate(path, size); if (res == -1) return -errno; return 0; } #ifdef HAVE_UTIMENSAT int FailureInjectorFs::fifs_utimens( const char *path, const struct timespec ts[2], struct fuse_file_info *fi) { (void) fi; int res; int injected_error = 0; if (CheckForInjectedError(path,"UTIMENS", &injected_error)) { cout << "Returning Injected error " << injected_error << "\n"; return -injected_error; } /* don't use utime/utimes since they follow symlinks */ res = utimensat(0, path, ts, AT_SYMLINK_NOFOLLOW); if (res == -1) return -errno; return 0; } #endif int FailureInjectorFs::fifs_create( const char *path, mode_t mode, struct fuse_file_info *fi) { int res; int injected_error = 0; if (CheckForInjectedError(path,"CREATE", &injected_error)) { cout << "Returning Injected error " << injected_error << "\n"; return -injected_error; } res = open(path, fi->flags, mode); if (res == -1) return -errno; fi->fh = res; return 0; } int FailureInjectorFs::fifs_open( const char *path, struct fuse_file_info *fi) { int res; int injected_error = 0; if (CheckForInjectedError(path,"OPEN", &injected_error)) { cout << "Returning Injected error " << injected_error << "\n"; return -injected_error; } res = open(path, fi->flags); if (res == -1) return -errno; fi->fh = res; return 0; } bool FailureInjectorFs::CheckForInjectedError( string path, string op, int *injected_error) { char *mpath = strdup(path.c_str()); if (mFailureInjector == NULL) { return false; } auto failures = mFailureInjector->GetFailures(path, op); while (failures == NULL) { // Check if failures are injected in any of the parent directories. char *dir = dirname(mpath); failures = mFailureInjector->GetFailures(string(dir), op); if ((failures == NULL) && (!strcmp(dir, ".") || !strcmp(dir, "/"))) { free(mpath); return false; } mpath = dir; } free(mpath); *injected_error = 0; useconds_t delay = 1; for (auto f : *failures) { switch(f.code) { case InjectedAction::DELAY: delay = f.delay * MICROSECONDS_IN_A_SECOND; break; case InjectedAction::FAIL: if (*injected_error != CORRUPT_DATA_ERROR_CODE) { *injected_error = f.error_code; } break; case InjectedAction::CORRUPT: *injected_error = CORRUPT_DATA_ERROR_CODE; break; default: // Ignore for now. break; } } // First create the delay. usleep(delay); if (*injected_error) { return true; } return false; } void FailureInjectorFs::FillCorruptData( char *buf, size_t size) { // For now just fill with some pattern based on input. // TBD : Fill with random data pattern. for (size_t i = 0; i < size; ++ i) { buf[i] = buf[i] + 2; } } int FailureInjectorFs::fifs_read( const char *path, char *buf, size_t size, off_t offset, struct fuse_file_info *fi) { int fd; int res; int injected_error = 0; if (CheckForInjectedError(path, "READ", &injected_error)) { cout << "Returning Injected error " << injected_error << "\n"; if (injected_error != CORRUPT_DATA_ERROR_CODE) { return -injected_error; } } if(fi == NULL) fd = open(path, O_RDONLY); else fd = fi->fh; if (fd == -1) return -errno; res = pread(fd, buf, size, offset); if (res == -1) res = -errno; if(fi == NULL) close(fd); if ((res != -1) && (injected_error == CORRUPT_DATA_ERROR_CODE)) { FillCorruptData(buf, size); return res; } return res; } int FailureInjectorFs::fifs_write( const char *path, const char *buf, size_t size, off_t offset, struct fuse_file_info *fi) { int fd; int res; int injected_error = 0; (void) fi; if (CheckForInjectedError(path, "WRITE", &injected_error)) { cout << "Returning Injected error " << injected_error << "\n"; if (injected_error != CORRUPT_DATA_ERROR_CODE) { return -injected_error; } else { FillCorruptData(const_cast(buf), size); } } if(fi == NULL) fd = open(path, O_WRONLY); else fd = fi->fh; if (fd == -1) return -errno; res = pwrite(fd, buf, size, offset); if (res == -1) res = -errno; if(fi == NULL) close(fd); return res; } int FailureInjectorFs::fifs_statfs( const char *path, struct statvfs *stbuf) { int res; int injected_error = 0; if (CheckForInjectedError(path, "STATFS", &injected_error)) { cout << "Returning Injected error " << injected_error << "\n"; return -injected_error; } res = statvfs(path, stbuf); if (res == -1) return -errno; return 0; } int FailureInjectorFs::fifs_release( const char *path, struct fuse_file_info *fi) { (void) path; close(fi->fh); return 0; } int FailureInjectorFs::fifs_fsync( const char *path, int isdatasync, struct fuse_file_info *fi) { int injected_error = 0; if (CheckForInjectedError(path,"FSYNC", &injected_error)) { cout << "Returning Injected error " << injected_error << "\n"; return -injected_error; } (void) path; (void) isdatasync; (void) fi; return 0; } #ifdef HAVE_POSIX_FALLOCATE int FailureInjectorFs::fifs_fallocate( const char *path, int mode, off_t offset, off_t length, struct fuse_file_info *fi) { int fd; int res; int injected_error = 0; (void) fi; if (CheckForInjectedError(path,"FALLOCATE", &injected_error)) { cout << "Returning Injected error " << injected_error << "\n"; return -injected_error; } if (mode) return -EOPNOTSUPP; if(fi == NULL) fd = open(path, O_WRONLY); else fd = fi->fh; if (fd == -1) return -errno; res = -posix_fallocate(fd, offset, length); if(fi == NULL) close(fd); return res; } #endif #ifdef HAVE_SETXATTR /* xattr operations are optional and can safely be left unimplemented */ int FailureInjectorFs::fifs_setxattr( const char *path, const char *name, const char *value, size_t size, int flags) { int injected_error = 0; if (CheckForInjectedError(path,"SETXATTR", &injected_error)) { cout << "Returning Injected error " << injected_error << "\n"; return -injected_error; } int res = lsetxattr(path, name, value, size, flags); if (res == -1) return -errno; return 0; } int FailureInjectorFs::fifs_getxattr( const char *path, const char *name, char *value, size_t size) { int injected_error = 0; if (CheckForInjectedError(path,"GETXATTR", &injected_error)) { cout << "Returning Injected error " << injected_error << "\n"; return -injected_error; } int res = lgetxattr(path, name, value, size); if (res == -1) return -errno; return res; } int FailureInjectorFs::fifs_listxattr( const char *path, char *list, size_t size) { int injected_error = 0; if (CheckForInjectedError(path,"LISTXATTR", &injected_error)) { cout << "Returning Injected error " << injected_error << "\n"; return -injected_error; } int res = llistxattr(path, list, size); if (res == -1) return -errno; return res; } int FailureInjectorFs::fifs_removexattr( const char *path, const char *name) { int injected_error = 0; if (CheckForInjectedError(path,"REMOVEXATTR", &injected_error)) { cout << "Returning Injected error " << injected_error << "\n"; return -injected_error; } int res = lremovexattr(path, name); if (res == -1) return -errno; return 0; } #endif /* HAVE_SETXATTR */ off_t FailureInjectorFs::fifs_lseek( const char *path, off_t off, int whence, struct fuse_file_info *fi) { int fd; off_t res; int injected_error = 0; if (CheckForInjectedError(path,"LSEEK", &injected_error)) { cout << "Returning Injected error " << injected_error << "\n"; return -injected_error; } if (fi == NULL) fd = open(path, O_RDONLY); else fd = fi->fh; if (fd == -1) return -errno; res = lseek(fd, off, whence); if (res == -1) res = -errno; if (fi == NULL) close(fd); return res; } void FailureInjectorFs::load_operations() { mFuseOperationsVec.getattr = FailureInjectorFs::fifs_getattr; mFuseOperationsVec.readlink = FailureInjectorFs::fifs_readlink; mFuseOperationsVec.mknod = FailureInjectorFs::fifs_mknod; mFuseOperationsVec.mkdir = FailureInjectorFs::fifs_mkdir; mFuseOperationsVec.unlink = FailureInjectorFs::fifs_unlink; mFuseOperationsVec.rmdir = FailureInjectorFs::fifs_rmdir; mFuseOperationsVec.symlink = FailureInjectorFs::fifs_symlink; mFuseOperationsVec.rename = FailureInjectorFs::fifs_rename; mFuseOperationsVec.link = FailureInjectorFs::fifs_link; mFuseOperationsVec.chmod = FailureInjectorFs::fifs_chmod; mFuseOperationsVec.chown = FailureInjectorFs::fifs_chown; mFuseOperationsVec.truncate = FailureInjectorFs::fifs_truncate; mFuseOperationsVec.open = FailureInjectorFs::fifs_open; mFuseOperationsVec.read = FailureInjectorFs::fifs_read; mFuseOperationsVec.write = FailureInjectorFs::fifs_write; mFuseOperationsVec.statfs = FailureInjectorFs::fifs_statfs; mFuseOperationsVec.flush = NULL; mFuseOperationsVec.release = FailureInjectorFs::fifs_release; mFuseOperationsVec.fsync = FailureInjectorFs::fifs_fsync; #ifdef HAVE_SETXATTR mFuseOperationsVec.setxattr = FailureInjectorFs::fifs_setxattr; mFuseOperationsVec.getxattr = FailureInjectorFs::fifs_getxattr; mFuseOperationsVec.listxattr = FailureInjectorFs::fifs_listxattr; mFuseOperationsVec.removexattr = FailureInjectorFs::fifs_removexattr; #else mFuseOperationsVec.setxattr = NULL; mFuseOperationsVec.getxattr = NULL; mFuseOperationsVec.listxattr = NULL; mFuseOperationsVec.removexattr = NULL; #endif mFuseOperationsVec.opendir = NULL; mFuseOperationsVec.readdir = FailureInjectorFs::fifs_readdir; mFuseOperationsVec.releasedir = NULL; mFuseOperationsVec.fsyncdir = NULL; mFuseOperationsVec.init = FailureInjectorFs::fifs_init; mFuseOperationsVec.destroy = NULL; mFuseOperationsVec.access = FailureInjectorFs::fifs_access; mFuseOperationsVec.create = FailureInjectorFs::fifs_create; mFuseOperationsVec.lock = NULL; #ifdef HAVE_UTIMENSAT mFuseOperationsVec.utimens = FailureInjectorFs::fifs_utimens; #else mFuseOperationsVec.utimens = NULL; #endif mFuseOperationsVec.bmap = NULL; mFuseOperationsVec.ioctl = NULL; mFuseOperationsVec.poll = NULL; mFuseOperationsVec.write_buf = NULL; mFuseOperationsVec.read_buf = NULL; mFuseOperationsVec.flock = NULL; #ifdef HAVE_POSIX_FALLOCATE mFuseOperationsVec.fallocate = FailureInjectorFs::fifs_fallocate; #else mFuseOperationsVec.fallocate = NULL; #endif mFuseOperationsVec.copy_file_range = NULL; } const struct fuse_operations *FailureInjectorFs::getOperations() { return &mFuseOperationsVec; } int main(int argc, char *argv[]) { FailureInjector *injector = new FailureInjector(); FailureInjectorFs noise_fs(injector); noise_fs.load_operations(); umask(0); fuse_main(argc, argv, noise_fs.getOperations(), NULL); }