From 791f42c10474cd26f096c070a0feba51e4c72706 Mon Sep 17 00:00:00 2001 From: Christian Thaeter Date: Sun, 30 Mar 2008 04:31:32 +0200 Subject: [PATCH] WIP: Low level file management We have 'File' which associates a name with a file and will be used to offer an interface for the application. Posix 'Filehandles' which are managed in a 'Filehandlecache' to utilize a limited number of fd's at optimum. Internally we use 'Filedescriptor's which manage files, mapping and all kinds of metadata. Filedescriptors weak referenced by a registry and freed when the last user vanishes. --- src/backend/Makefile.am | 15 ++- src/backend/file.c | 64 +++++++++++ src/backend/file.h | 90 +++++++++++++++ src/backend/filedescriptor.c | 199 ++++++++++++++++++++++++++++++++++ src/backend/filedescriptor.h | 58 ++++++++++ src/backend/filehandle.h | 51 +++++++++ src/backend/filehandlecache.c | 83 ++++++++++++++ src/backend/filehandlecache.h | 97 +++++++++++++++++ 8 files changed, 649 insertions(+), 8 deletions(-) create mode 100644 src/backend/file.c create mode 100644 src/backend/file.h create mode 100644 src/backend/filedescriptor.c create mode 100644 src/backend/filedescriptor.h create mode 100644 src/backend/filehandle.h create mode 100644 src/backend/filehandlecache.c create mode 100644 src/backend/filehandlecache.h diff --git a/src/backend/Makefile.am b/src/backend/Makefile.am index eb5853d5f..45ce555a8 100644 --- a/src/backend/Makefile.am +++ b/src/backend/Makefile.am @@ -31,12 +31,11 @@ liblumibackend_a_SOURCES = \ # $(liblumibackend_a_srcdir)/filehandlecache.c -noinst_HEADERS += \ - $(liblumibackend_a_srcdir)/mediaaccessfacade.cpp - -# $(liblumibackend_a_srcdir)/backend.h \ -# $(liblumibackend_a_srcdir)/file.h \ -# $(liblumibackend_a_srcdir)/filehandle.h \ -# $(liblumibackend_a_srcdir)/filedescriptor.h \ -# $(liblumibackend_a_srcdir)/filehandlecache.h +noinst_HEADERS += \ + $(liblumibackend_a_srcdir)/mediaaccessfacade.cpp \ + $(liblumibackend_a_srcdir)/backend.h \ + $(liblumibackend_a_srcdir)/file.h \ + $(liblumibackend_a_srcdir)/filehandle.h \ + $(liblumibackend_a_srcdir)/filedescriptor.h \ + $(liblumibackend_a_srcdir)/filehandlecache.h diff --git a/src/backend/file.c b/src/backend/file.c new file mode 100644 index 000000000..7a4aa5050 --- /dev/null +++ b/src/backend/file.c @@ -0,0 +1,64 @@ +/* + file.c - file handling + + Copyright (C) Lumiera.org + 2008, Christian Thaeter + + This program is free software; you can redistribute it and/or + modify it under the terms of the GNU General Public License as + published by the Free Software Foundation; either version 2 of the + License, or (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, write to the Free Software + Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA. +*/ + +#include "lib/mutex.h" +#include "lib/safeclib.h" + +#include "backend/file.h" +#include "backend/filedescriptor.h" + +#include + + +NOBUG_DEFINE_FLAG (lumiera_file); + +LumieraFile +lumiera_file_init (LumieraFile self, const char* name, int flags) +{ + if (!lumiera_filedescriptor_acquire (&self->descriptor, name, flags)) + return NULL; + self->name = lumiera_strndup (name, PATH_MAX); + + return self; +} + +LumieraFile +lumiera_file_destroy (LumieraFile self) +{ + lumiera_reference_destroy (&self->descriptor); + free ((void*)self->name); + return self; +} + + +LumieraFile +lumiera_file_new (const char* name, int flags) +{ + LumieraFile self = lumiera_malloc (sizeof (lumiera_file)); + return lumiera_file_init (self, name, flags); +} + +void +lumiera_file_delete (LumieraFile self) +{ + free (lumiera_file_destroy (self)); +} + diff --git a/src/backend/file.h b/src/backend/file.h new file mode 100644 index 000000000..5018d321a --- /dev/null +++ b/src/backend/file.h @@ -0,0 +1,90 @@ +/* + file.h - interface to files by filename + + Copyright (C) Lumiera.org + 2008, Christian Thaeter + + This program is free software; you can redistribute it and/or + modify it under the terms of the GNU General Public License as + published by the Free Software Foundation; either version 2 of the + License, or (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, write to the Free Software + Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA. +*/ + +#ifndef LUMIERA_FILE_H +#define LUMIERA_FILE_H + + +#include "lib/mutex.h" +#include "lib/llist.h" +#include "lib/error.h" +#include "lib/references.h" + +/** + * @file File management + * Handling Files is splitted into different classes: + * 1. The 'lumiera_file' class which acts as interface to the outside for managing files. + * 'lumiera_file' is addressed by the name of the file. Since files can have more than one name (hardlinks) + * many 'lumiera_file' can point to a single 'lumiera_filedescriptor' + * 2. The 'lumiera_filedescriptor' class which does the real work managing the file in the back. + * 3. Since OS-filehandles are a limited resource we access the lazily as 'lumiera_filehandle' which are + * managed in a 'lumiera_filehandlecache' + */ + +typedef struct lumiera_file_struct lumiera_file; +typedef lumiera_file* LumieraFile; + + +#include "backend/filehandle.h" +#include "backend/filedescriptor.h" + + + +NOBUG_DECLARE_FLAG (lumiera_file); + +#define LUMIERA_FILE_READONLY (O_RDONLY | O_LARGEFILE | O_NOATIME) +#define LUMIERA_FILE_READWRITE (O_RDWR | O_LARGEFILE | O_NOATIME) +#define LUMIERA_FILE_CREATE (O_RDWR | O_LARGEFILE | O_NOATIME | O_CREAT | O_EXCL) +#define LUMIERA_FILE_RECREATE (O_RDWR | O_LARGEFILE | O_NOATIME | O_CREAT | O_TRUNC) + +struct lumiera_file_struct +{ + const char* name; + struct lumiera_reference_struct descriptor; +}; + +/** + * + */ +LumieraFile +lumiera_file_init (LumieraFile self, const char* name, int flags); + +/** + * + */ +LumieraFile +lumiera_file_destroy (LumieraFile self); + +/** + * + */ +LumieraFile +lumiera_file_new (const char* name, int flags); + +/** + * + */ +void +lumiera_file_delete (LumieraFile self); + + +#endif + diff --git a/src/backend/filedescriptor.c b/src/backend/filedescriptor.c new file mode 100644 index 000000000..fdb22ffb3 --- /dev/null +++ b/src/backend/filedescriptor.c @@ -0,0 +1,199 @@ +/* + filedescriptor.c - file handling + + Copyright (C) Lumiera.org + 2008, Christian Thaeter + + This program is free software; you can redistribute it and/or + modify it under the terms of the GNU General Public License as + published by the Free Software Foundation; either version 2 of the + License, or (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, write to the Free Software + Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA. +*/ + +#include "lib/mutex.h" +#include "lib/safeclib.h" +#include "lib/cuckoo.h" + +#include "backend/filedescriptor.h" + +#include +#include +#include +#include + + +NOBUG_DEFINE_FLAG (filedescriptor); + +static Cuckoo registry = NULL; +static lumiera_mutex registry_mutex = {PTHREAD_MUTEX_INITIALIZER}; + + +static size_t +h1 (const void* item, const uint32_t r) +{ + const LumieraFiledescriptor i = (const LumieraFiledescriptor) item; + return i->stat.st_dev^i->stat.st_ino^i->flags^((i->stat.st_dev^i->stat.st_ino^i->flags)>>7)^r; +} + +static size_t +h2 (const void* item, const uint32_t r) +{ + const LumieraFiledescriptor i = (const LumieraFiledescriptor) item; + return i->stat.st_dev^i->stat.st_ino^i->flags^((i->stat.st_dev^i->stat.st_ino^i->flags)>>5)^r; +} + +static size_t +h3 (const void* item, const uint32_t r) +{ + const LumieraFiledescriptor i = (const LumieraFiledescriptor) item; + return i->stat.st_dev^i->stat.st_ino^i->flags^((i->stat.st_dev^i->stat.st_ino^i->flags)>>3)^r; +} + + +static int +cmp (const void* keya, const void* keyb) +{ + const LumieraFiledescriptor a = (const LumieraFiledescriptor) keya; + const LumieraFiledescriptor b = (const LumieraFiledescriptor) keyb; + return a->stat.st_dev == b->stat.st_dev && a->stat.st_ino == b->stat.st_ino && a->flags == b->flags; +} + + +static Cuckoo +lumiera_filedescriptor_registry_new (void) +{ + NOBUG_INIT_FLAG (filedescriptor); + + Cuckoo registry = cuckoo_new (h1, h2, h3, cmp, + sizeof (struct lumiera_reference_struct), + 3); + if (!registry) + LUMIERA_DIE (NO_MEMORY); + + return registry; +} + +/** + * Find existing filedescriptor or create one + * @param descriptor strong reference to be initialized with the filedescriptor, + * a filedescriptor will be deleted when its last string reference gets deleted. + * @param name name of the file + * @param flags opening flags for the filedescriptor + * @return descriptor on success or NULL on error + */ +LumieraReference +lumiera_filedescriptor_acquire (LumieraReference descriptor, const char* name, int flags) +{ + UNCHECKED; + lumiera_mutexacquirer registry_lock; + lumiera_mutexacquirer_init_mutex (®istry_lock, ®istry_mutex, LUMIERA_LOCKED); + + if (!registry) + registry = lumiera_filedescriptor_registry_new (); + + struct lumiera_filedescriptor_struct findme; + findme.flags = flags; + + for (int retry = 0; !retry; ++retry) + { + if (stat (name, &findme.stat) == 0) + break; + else + { + if (errno == ENOENT && flags&O_CREAT) + { + TRACE (filedescriptor, "try creating file: %s", name); + char* slash = lumiera_tmpbuf_strndup (name, PATH_MAX); + + while ((slash = strchr (slash+1, '/'))) + { + *slash = '\0'; + if (mkdir (name, 0777) == -1) + { + LUMIERA_ERROR_SET (filedescriptor, ERRNO); + goto efile; + } + *slash = '/'; + } + + int fd; + fd = creat (name, 0777); + if (fd == -1) + { + LUMIERA_ERROR_SET (filedescriptor, ERRNO); + goto efile; + } + close (fd); + /* will now retry the fstat once */ + } + else + { + LUMIERA_ERROR_SET (filedescriptor, ERRNO); + goto efile; + } + } + } + + /* lookup/create descriptor */ + LumieraReference elem = cuckoo_find (registry, &findme); + + if (!elem) + { + TRACE (filedescriptor, "Descriptor not found"); + + lumiera_reference_strong_init_once (descriptor, + lumiera_filedescriptor_new (&findme), + (void(*)(void*))lumiera_filedescriptor_delete); + + if (!lumiera_reference_get (descriptor)) + goto ecreate; + + lumiera_reference weak_descriptor; + lumiera_reference_weak_init (&weak_descriptor, descriptor); + cuckoo_insert (registry, &weak_descriptor); + } + else + { + lumiera_reference_strong_init (descriptor, elem); + } + lumiera_mutexacquirer_unlock (®istry_lock); + + return descriptor; + + efile: + ecreate: + lumiera_mutexacquirer_unlock (®istry_lock); + return NULL; +} + + +/** + * Allocate a new filedescriptor cloned from a template + * @param template the source filedescriptor + * @return the constrccted filedescriptor + */ +LumieraFiledescriptor +lumiera_filedescriptor_new (LumieraFiledescriptor template) +{ + UNIMPLEMENTED(""); + return NULL; +} + +/** + * delete a filedescriptor and free its resources + * @param descriptor filedescriptor to be freed + */ +void +lumiera_filedescriptor_delete (LumieraFiledescriptor descriptor) +{ + UNIMPLEMENTED(""); +} diff --git a/src/backend/filedescriptor.h b/src/backend/filedescriptor.h new file mode 100644 index 000000000..060839d0a --- /dev/null +++ b/src/backend/filedescriptor.h @@ -0,0 +1,58 @@ +/* + filedescriptor.h - file handling + + Copyright (C) Lumiera.org + 2008, Christian Thaeter + + This program is free software; you can redistribute it and/or + modify it under the terms of the GNU General Public License as + published by the Free Software Foundation; either version 2 of the + License, or (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, write to the Free Software + Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA. +*/ + +#ifndef LUMIERA_FILEDESCRIPTOR_H +#define LUMIERA_FILEDESCRIPTOR_H + +#include "lib/mutex.h" + +#include + +typedef struct lumiera_filedescriptor_struct lumiera_filedescriptor; +typedef lumiera_filedescriptor* LumieraFiledescriptor; + + +#include "backend/filehandle.h" +#include "backend/file.h" + + +struct lumiera_filedescriptor_struct +{ + struct stat stat; /* create after first open, maintained metadata, MUST BE FIRST! */ + int flags; /* open flags, must be masked for reopen */ + lumiera_mutex lock; /* locks operations on this file descriptor */ + unsigned reopened; /* count for reopens 0=not yet opened, 1=first, 2..=reopened*/ + //LumieraFileMap mappings; + //LumieraWriteBuffer writebuffer; +}; + + +LumieraReference +lumiera_filedescriptor_acquire (LumieraReference descriptor, const char* name, int flags); + +LumieraFiledescriptor +lumiera_filedescriptor_new (LumieraFiledescriptor template); + +void +lumiera_filedescriptor_delete (LumieraFiledescriptor self); + + +#endif diff --git a/src/backend/filehandle.h b/src/backend/filehandle.h new file mode 100644 index 000000000..00b4737c9 --- /dev/null +++ b/src/backend/filehandle.h @@ -0,0 +1,51 @@ +/* + filehandle - filehandle management and caching + + Copyright (C) Lumiera.org + 2008, Christian Thaeter + + This program is free software; you can redistribute it and/or + modify it under the terms of the GNU General Public License as + published by the Free Software Foundation; either version 2 of the + License, or (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, write to the Free Software + Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA. +*/ +#ifndef LUMIERA_FILEHANDLE_H +#define LUMIERA_FILEHANDLE_H + +#include "lib/error.h" +#include "lib/llist.h" + +/** + * @file Filehandles + */ + + +/** + * File handles + */ +struct lumiera_filehandle_struct +{ + llist cachenode; + int handle; + //LumieraFile file; + //lumiera_mutex lock; +}; +typedef struct lumiera_filehandle_struct lumiera_filehandle; +typedef lumiera_filehandle* LumieraFilehandle; + +LumieraFilehandle +lumiera_filehandle_new (); + +void* +lumiera_filehandle_destroy_node (LList node); + +#endif diff --git a/src/backend/filehandlecache.c b/src/backend/filehandlecache.c new file mode 100644 index 000000000..98ebdddc9 --- /dev/null +++ b/src/backend/filehandlecache.c @@ -0,0 +1,83 @@ +/* + filehandlecache - filehandle management and caching + + Copyright (C) Lumiera.org + 2008, Christian Thaeter + + This program is free software; you can redistribute it and/or + modify it under the terms of the GNU General Public License as + published by the Free Software Foundation; either version 2 of the + License, or (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, write to the Free Software + Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA. +*/ + +#include "backend/filehandlecache.h" + +NOBUG_DEFINE_FLAG (lumiera_filehandlecache); + +/* errors */ + +LUMIERA_ERROR_DEFINE (FILEHANDLECACHE_NOHANDLE, "No filehandle available"); + + +/**/ + +LumieraFilehandlecache +lumiera_filehandlecache_init (LumieraFilehandlecache self, int max_entries) +{ + lumiera_mrucache_init (&self->cache, lumiera_filehandle_destroy_node); + self->available =max_entries; + lumiera_mutex_init (&self->lock); + return self; +} + +LumieraFilehandlecache +lumiera_filehandlecache_destroy (LumieraFilehandlecache self) +{ + self->available += self->cache.cached; + lumiera_mrucache_destroy (&self->cache); + lumiera_mutex_destroy (&self->lock); + return self; +} + + +LumieraFilehandle +lumiera_filehandlecache_filehandle_acquire (LumieraFilehandlecache self) +{ + LumieraFilehandle ret; + LUMIERA_MUTEX_SECTION (&self->lock) + { + if (self->available <= 0 && self->cache.cached) + { + /* pop a filehandle from cache */ + ret = lumiera_mrucache_pop (&self->cache); + if (self->available < 0) + /* try to free overallocated filehandles */ + self->available -= self->available + lumiera_mrucache_age (&self->cache, -self->available); + } + else + { + /* allocate new filehandle if we are below the limit or no cached handles where available (overallocating) */ + ret = lumiera_filehandle_new (); + if (!ret) + LUMIERA_ERROR_SET (lumiera_filehandlecache, FILEHANDLECACHE_NOHANDLE); + else + --self->available; + } + } + return ret; +} + +void +lumiera_filehandlecache_add (LumieraFilehandlecache self, LumieraFilehandle handle) +{ + UNIMPLEMENTED (""); +} diff --git a/src/backend/filehandlecache.h b/src/backend/filehandlecache.h new file mode 100644 index 000000000..40386defe --- /dev/null +++ b/src/backend/filehandlecache.h @@ -0,0 +1,97 @@ +/* + filehandlecache - filehandle management and caching + + Copyright (C) Lumiera.org + 2008, Christian Thaeter + + This program is free software; you can redistribute it and/or + modify it under the terms of the GNU General Public License as + published by the Free Software Foundation; either version 2 of the + License, or (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, write to the Free Software + Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA. +*/ +#ifndef LUMIERA_FILEHANDLECACHE_H +#define LUMIERA_FILEHANDLECACHE_H + +#include "lib/error.h" +#include "lib/mrucache.h" +#include "lib/mutex.h" + +#include "backend/filehandle.h" + +/** + * @file Filehandle management and caching + * The number of filehandles a program can held open is usually limited, since we want to support + * using a less limited number of files and closing/opening for each operation is expensive, we + * provide a cache to keep the most frequent used files open and gracefully close/recycle unused filehandles. + */ + + + +/* + File handle cache manages file handles + */ +struct lumiera_filehandlecache_struct +{ + lumiera_mrucache cache; + int available; + lumiera_mutex lock; +}; +typedef struct lumiera_filehandlecache_struct lumiera_filehandlecache; +typedef lumiera_filehandlecache* LumieraFilehandlecache; + + +/** + * Initializes a filehandle cache. + * @param self pointer to the cache to be initialized + * @param max_entries number how much filehandles shall be managed + * @return self as given + * The number of elements the cache can hold is static and should be + * determined by sysconf (_SC_OPEN_MAX) minus some safety margin. + */ +LumieraFilehandlecache +lumiera_filehandlecache_init (LumieraFilehandlecache self, int max_entries); + +/** + * Destroy a filehandle cache. + * @param self the cache to be destroyed + * @return self as given to the now uninitialized! cache + * No filehandles in the cache must be locked, this would be a fatal error. + * The handles are closed automatically. + */ +LumieraFilehandlecache +lumiera_filehandlecache_destroy (LumieraFilehandlecache self); + +/** + * Get a fresh filehandle. + * @param self pointer to the cache + * @return the new filehandle + * one must call lumiera_mutexacquirer_unlock (&lock) to free the lock on the handle when done + */ +LumieraFilehandle +lumiera_filehandlecache_filehandle_acquire (LumieraFilehandlecache self); + +/** + * Add filehande back to cache. + * @param self pointer to the cache + * @param handle filehandle to be put back + */ +void +lumiera_filehandlecache_add (LumieraFilehandlecache self, LumieraFilehandle handle); + +/** + * + */ +LumieraFilehandle +lumiera_filehandlecache_remove (LumieraFilehandlecache self, int fd); + + +#endif