diff --git a/src/backend/Makefile.am b/src/backend/Makefile.am index fbfbed33e..e84d3c6fe 100644 --- a/src/backend/Makefile.am +++ b/src/backend/Makefile.am @@ -30,10 +30,12 @@ liblumierabackend_la_SOURCES = \ $(liblumierabackend_la_srcdir)/file.c \ $(liblumierabackend_la_srcdir)/filehandle.c \ $(liblumierabackend_la_srcdir)/filedescriptor.c \ + $(liblumierabackend_la_srcdir)/filedescriptorregistry.c \ $(liblumierabackend_la_srcdir)/filehandlecache.c \ $(liblumierabackend_la_srcdir)/mmap.c \ $(liblumierabackend_la_srcdir)/mmapings.c \ $(liblumierabackend_la_srcdir)/mmapcache.c \ + $(liblumierabackend_la_srcdir)/resourcecollector.c \ $(liblumierabackend_la_srcdir)/threadpool-init.cpp \ $(liblumierabackend_la_srcdir)/enginefacade.cpp \ $(liblumierabackend_la_srcdir)/scriptrunnerfacade.cpp \ @@ -47,8 +49,10 @@ noinst_HEADERS += \ $(liblumierabackend_la_srcdir)/file.h \ $(liblumierabackend_la_srcdir)/filehandle.h \ $(liblumierabackend_la_srcdir)/filedescriptor.h \ + $(liblumierabackend_la_srcdir)/filedescriptorregistry.h \ $(liblumierabackend_la_srcdir)/filehandlecache.h \ $(liblumierabackend_la_srcdir)/mmap.h \ $(liblumierabackend_la_srcdir)/mmapings.h \ - $(liblumierabackend_la_srcdir)/mmapcache.h + $(liblumierabackend_la_srcdir)/mmapcache.h \ + $(liblumierabackend_la_srcdir)/resourcecollector.h diff --git a/src/backend/backend.c b/src/backend/backend.c index 5440632a1..baa3d49b6 100644 --- a/src/backend/backend.c +++ b/src/backend/backend.c @@ -21,13 +21,16 @@ #include "include/logging.h" #include "lib/safeclib.h" +#include "lib/mpool.h" #include "backend/backend.h" #include "common/config.h" #include "backend/filehandlecache.h" #include "backend/filedescriptor.h" +#include "backend/filedescriptorregistry.h" #include "backend/mmapcache.h" #include "backend/threadpool.h" +#include "backend/resourcecollector.h" #include #include @@ -49,8 +52,20 @@ //NOBUG_DECLARE_FLAG (mmapcache); + +static enum lumiera_resource_try +lumiera_backend_mpool_purge (enum lumiera_resource_try itr, void* data, void* context); + +static void +lumiera_backend_resourcecollector_register_mpool (MPool self); + +static void +lumiera_backend_resourcecollector_unregister_mpool (MPool self); + + size_t lumiera_backend_pagesize; + int lumiera_backend_init (void) { @@ -64,16 +79,30 @@ lumiera_backend_init (void) //NOBUG_INIT_FLAG (mmapcache); TRACE (backend_dbg); + lumiera_mutex_init (&lumiera_filecreate_mutex, "fileaccess", &NOBUG_FLAG (mutex_dbg), NOBUG_CONTEXT); + + lumiera_resourcecollector_init (); + + /* hook the resourcecollector into the mpool*/ + mpool_malloc_hook = lumiera_malloc; + mpool_free_hook = lumiera_free; + mpool_init_hook = lumiera_backend_resourcecollector_register_mpool; + mpool_destroy_hook = lumiera_backend_resourcecollector_unregister_mpool; + + /* hook the resourcecollector into the safeclib allocation functions */ + lumiera_safeclib_set_resourcecollector (lumiera_resourcecollector_run); + + PLANNED("The resourcecollector aborts by default when there is no final strategy for recovery, TODO: initiate sane shutdown"); lumiera_threadpool_init (); + PLANNED ("hook threadpool into the resourcecollector (maybe in threadpool_init() instead here"); - lumiera_filedescriptor_registry_init (); + lumiera_filedescriptorregistry_init (); lumiera_backend_pagesize = sysconf(_SC_PAGESIZE); TODO ("add config options to override following defaults"); - const char* filehandles = lumiera_tmpbuf_snprintf (SIZE_MAX, "backend.file.max_handles = %d", /* roughly 2/3 of all availables filehandles are managed by the backend */ @@ -113,8 +142,54 @@ void lumiera_backend_destroy (void) { TRACE (backend_dbg); + lumiera_mmapcache_delete (); lumiera_filehandlecache_delete (); - lumiera_filedescriptor_registry_destroy (); + lumiera_filedescriptorregistry_destroy (); lumiera_threadpool_destroy (); + + lumiera_safeclib_set_resourcecollector (NULL); + + mpool_init_hook = NULL; + mpool_destroy_hook = NULL; + mpool_malloc_hook = malloc; + mpool_free_hook = free; + + lumiera_resourcecollector_destroy (); + + lumiera_mutex_destroy (&lumiera_filecreate_mutex, &NOBUG_FLAG (mutex_dbg), NOBUG_CONTEXT); } + + +static enum lumiera_resource_try +lumiera_backend_mpool_purge (enum lumiera_resource_try itr, void* data, void* context) +{ + (void) context; + (void) data; + (void) itr; + TODO("mpool_purge ((MPool) data)"); + return LUMIERA_RESOURCE_NONE; +} + +static void +lumiera_backend_resourcecollector_register_mpool (MPool self) +{ + self->udata = + lumiera_resourcecollector_register_handler (LUMIERA_RESOURCE_MEMORY, lumiera_backend_mpool_purge, self); +} + +static void +lumiera_backend_resourcecollector_unregister_mpool (MPool self) +{ + lumiera_resourcehandler_unregister ((LumieraResourcehandler) self->udata); +} + + + +/* +// Local Variables: +// mode: C +// c-file-style: "gnu" +// indent-tabs-mode: nil +// End: +*/ diff --git a/src/backend/backend.h b/src/backend/backend.h index 8bfa18db5..f1e68c85b 100644 --- a/src/backend/backend.h +++ b/src/backend/backend.h @@ -21,12 +21,21 @@ #ifndef LUMIERA_BACKEND_H #define LUMIERA_BACKEND_H +#include "lib/mutex.h" + #include //NOBUG_DECLARE_FLAG (backend); extern size_t lumiera_backend_pagesize; +/** + * Protect lookup and creation of files. + * Trying to access a nonexistent file with O_CREAT would be racy. + * Defined in filedescriptor.c + */ +extern lumiera_mutex lumiera_filecreate_mutex; + int lumiera_backend_init (void); diff --git a/src/backend/file.c b/src/backend/file.c index 8302f9149..185d11b78 100644 --- a/src/backend/file.c +++ b/src/backend/file.c @@ -54,11 +54,14 @@ lumiera_file_init (LumieraFile self, const char* name, int flags) } LumieraFile -lumiera_file_destroy (LumieraFile self) +lumiera_file_destroy (LumieraFile self, int do_unlink) { TRACE (file_dbg); lumiera_filedescriptor_release (self->descriptor, self->name, &self->node); + if (do_unlink) + unlink (self->name); + lumiera_free (self->name); return self; } @@ -85,7 +88,16 @@ lumiera_file_delete (LumieraFile self) { TRACE (file_dbg); TRACE (file, "close file '%s'", self->name); - lumiera_free (lumiera_file_destroy (self)); + lumiera_free (lumiera_file_destroy (self, 0)); +} + + +void +lumiera_file_delete_unlink (LumieraFile self) +{ + TRACE (file_dbg); + TRACE (file, "close and unlink file '%s'", self->name); + lumiera_free (lumiera_file_destroy (self, 1)); } @@ -124,6 +136,12 @@ lumiera_file_chunksize_set (LumieraFile self, size_t chunksize) size_t lumiera_file_chunksize_get (LumieraFile self) { + if (!self->descriptor->mmapings) + { + LUMIERA_ERROR_SET (file, FILE_NOCHUNKSIZE, lumiera_filedescriptor_name (self->descriptor)); + return 0; + } + return self->descriptor->mmapings->chunksize; } diff --git a/src/backend/file.h b/src/backend/file.h index cafd66939..91796998b 100644 --- a/src/backend/file.h +++ b/src/backend/file.h @@ -91,12 +91,11 @@ lumiera_file_init (LumieraFile self, const char* name, int flags); * Destroy a file structure. * frees all associated resources, releases the filedescriptor etc. * @param self file structure to be destroyed - * @param chunksize allocation/mmaping granularity, must be 2's exponent of pagesize - * only used at the first access to a file and ignored for subsequnet accesses + * @param do_unlink if 1 then delete the file physically from disk (only the associated name) * @return self */ LumieraFile -lumiera_file_destroy (LumieraFile self); +lumiera_file_destroy (LumieraFile self, int do_unlink); /** @@ -117,6 +116,14 @@ void lumiera_file_delete (LumieraFile self); +/** + * Frees a file structure and deletes the associated file name from disk. + * @param self file structure to be freed + */ +void +lumiera_file_delete_unlink (LumieraFile self); + + /** * Get a POSIX filehandle for a file. * Filehandles are opened on demand and must be acquired for use. diff --git a/src/backend/filedescriptor.c b/src/backend/filedescriptor.c index 849b2cd84..a0870e4eb 100644 --- a/src/backend/filedescriptor.c +++ b/src/backend/filedescriptor.c @@ -25,6 +25,7 @@ #include "backend/file.h" #include "backend/filedescriptor.h" +#include "backend/filedescriptorregistry.h" #include "backend/filehandle.h" #include "backend/filehandlecache.h" @@ -37,94 +38,19 @@ //NOBUG_DEFINE_FLAG_PARENT (filedescriptor, file_all); -/* - Filedescriptor registry - - This registry stores all acquired filedescriptors for lookup, they will be freed when not referenced anymore. - */ -static PSplay registry = NULL; -static lumiera_mutex registry_mutex; - - -static int -cmp_fn (const void* keya, const void* keyb) -{ - const LumieraFiledescriptor a = (const LumieraFiledescriptor)keya; - const LumieraFiledescriptor b = (const LumieraFiledescriptor)keyb; - - if (a->stat.st_dev < b->stat.st_dev) - return -1; - else if (a->stat.st_dev > b->stat.st_dev) - return 1; - - if (a->stat.st_ino < b->stat.st_ino) - return -1; - else if (a->stat.st_ino > b->stat.st_ino) - return 1; - - if ((a->flags&LUMIERA_FILE_MASK) < (b->flags&LUMIERA_FILE_MASK)) - return -1; - else if ((a->flags&LUMIERA_FILE_MASK) > (b->flags&LUMIERA_FILE_MASK)) - return 1; - - return 0; -} - - -static void -delete_fn (PSplaynode node) -{ - TODO ("figure name out? or is the handle here always closed"); - lumiera_filedescriptor_delete ((LumieraFiledescriptor) node, NULL); -} - - -static const void* -key_fn (const PSplaynode node) -{ - return node; -} - - - -void -lumiera_filedescriptor_registry_init (void) -{ - //NOBUG_INIT_FLAG (filedescriptor); - TRACE (filedescriptor_dbg); - REQUIRE (!registry); - - registry = psplay_new (cmp_fn, key_fn, delete_fn); - if (!registry) - LUMIERA_DIE (NO_MEMORY); - - lumiera_mutex_init (®istry_mutex, "filedescriptor_registry", &NOBUG_FLAG (mutex_dbg), NOBUG_CONTEXT); -} - -void -lumiera_filedescriptor_registry_destroy (void) -{ - TRACE (filedescriptor_dbg); - REQUIRE (!psplay_nelements (registry)); - - lumiera_mutex_destroy (®istry_mutex, &NOBUG_FLAG (mutex_dbg), NOBUG_CONTEXT); - - if (registry) - psplay_destroy (registry); - registry = NULL; -} +/* lookup and creation of files, initialized in backend.c */ +lumiera_mutex lumiera_filecreate_mutex; LumieraFiledescriptor lumiera_filedescriptor_acquire (const char* name, int flags, LList filenode) { TRACE (filedescriptor_dbg, "%s", name); - REQUIRE (registry, "not initialized"); REQUIRE (llist_is_empty (filenode)); LumieraFiledescriptor dest = NULL; - LUMIERA_MUTEX_SECTION (mutex_sync, ®istry_mutex) + LUMIERA_MUTEX_SECTION (mutex_sync, &lumiera_filecreate_mutex) { lumiera_filedescriptor fdesc; fdesc.flags = flags; @@ -167,25 +93,21 @@ lumiera_filedescriptor_acquire (const char* name, int flags, LList filenode) } /* lookup/create descriptor */ - dest = (LumieraFiledescriptor) psplay_find (registry, &fdesc, 100); + dest = lumiera_filedescriptorregistry_ensure (&fdesc); - if (!dest) - { - dest = lumiera_filedescriptor_new (&fdesc); - if (!dest) - goto error; + if (dest) + llist_insert_head (&dest->files, filenode); - psplay_insert (registry, &dest->node, 100); - } - - llist_insert_head (&dest->files, filenode); - error: ; + error: + ; } return dest; } + + void lumiera_filedescriptor_release (LumieraFiledescriptor self, const char* name, LList filenode) { @@ -288,29 +210,25 @@ lumiera_filedescriptor_delete (LumieraFiledescriptor self, const char* name) { TRACE (filedescriptor_dbg, "%p %s", self, name); - LUMIERA_MUTEX_SECTION (mutex_sync, ®istry_mutex) + REQUIRE (llist_is_empty (&self->files)); + + lumiera_filedescriptorregistry_remove (self); + + TODO ("destruct other members (WIP)"); + + lumiera_mmapings_delete (self->mmapings); + + if (self->handle && name && ((self->flags & O_RDWR) == O_RDWR)) { - REQUIRE (llist_is_empty (&self->files)); - - psplay_remove (registry, &self->node); - - TODO ("destruct other members (WIP)"); - - if (self->handle && name && ((self->flags & O_RDWR) == O_RDWR)) - { - TRACE (filedescriptor_dbg, "truncate %s to %lld", name, (long long)self->realsize); - lumiera_filehandlecache_checkout (self->handle); - int dummy = ftruncate (lumiera_filehandle_handle (self->handle), self->realsize); - (void) dummy; /* this is present to silence a warning */ - TODO ("handle error case better"); - lumiera_filehandlecache_checkin (self->handle); - } - - lumiera_mmapings_delete (self->mmapings); - - TODO ("release filehandle"); - - lumiera_mutex_destroy (&self->lock, &NOBUG_FLAG (mutex_dbg), NOBUG_CONTEXT); - lumiera_free (self); + TRACE (filedescriptor_dbg, "truncate %s to %lld", name, (long long)self->realsize); + lumiera_filehandlecache_checkout (self->handle); + int dummy = ftruncate (lumiera_filehandle_handle (self->handle), self->realsize); + (void) dummy; /* this is present to silence a warning */ + TODO ("handle error case better"); + lumiera_filehandlecache_checkin (self->handle); + TODO ("really release filehandle"); } + + lumiera_mutex_destroy (&self->lock, &NOBUG_FLAG (mutex_dbg), NOBUG_CONTEXT); + lumiera_free (self); } diff --git a/src/backend/filedescriptor.h b/src/backend/filedescriptor.h index a74e8c47d..b06839c1b 100644 --- a/src/backend/filedescriptor.h +++ b/src/backend/filedescriptor.h @@ -75,22 +75,6 @@ struct lumiera_filedescriptor_struct llist files; }; -/** - * Initialize the global filedescriptor registry. - * All filedescriptors are looked up by dev/ino/open-flags in a hashtable. Opening hardlinked - * files will be targeted to the same filedescriptor. - * This function never fails but dies on error. - */ -void -lumiera_filedescriptor_registry_init (void); - -/** - * Destroy and free the global filedescriptor registry. - * Never fails. - */ -void -lumiera_filedescriptor_registry_destroy (void); - /** * Find existing filedescriptor or create one diff --git a/src/backend/filedescriptorregistry.c b/src/backend/filedescriptorregistry.c new file mode 100644 index 000000000..557be2da1 --- /dev/null +++ b/src/backend/filedescriptorregistry.c @@ -0,0 +1,150 @@ +/* + filedescriptorregistry.c - register all files in use + + Copyright (C) Lumiera.org + 2008, 2010, Christian Thaeter + + This program is free software; you can redistribute it and/or + modify it under the terms of the GNU General Public License as + published by the Free Software Foundation; either version 2 of the + License, or (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, write to the Free Software + Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA. +*/ + +#include "include/logging.h" +#include "lib/safeclib.h" +#include "lib/mutex.h" +#include "lib/psplay.h" + +#include "backend/file.h" +#include "backend/filedescriptor.h" +#include "backend/filedescriptorregistry.h" + +//NOBUG_DEFINE_FLAG_PARENT (filedescriptorregistry, file_all); + +/* + Filedescriptor registry + + This registry stores all acquired filedescriptors for lookup, they will be freed when not referenced anymore. + */ +static PSplay filedescriptorregistry = NULL; +static lumiera_mutex filedescriptorregistry_mutex; + + +static int +cmp_fn (const void* keya, const void* keyb) +{ + const LumieraFiledescriptor a = (const LumieraFiledescriptor)keya; + const LumieraFiledescriptor b = (const LumieraFiledescriptor)keyb; + + if (a->stat.st_dev < b->stat.st_dev) + return -1; + else if (a->stat.st_dev > b->stat.st_dev) + return 1; + + if (a->stat.st_ino < b->stat.st_ino) + return -1; + else if (a->stat.st_ino > b->stat.st_ino) + return 1; + + if ((a->flags&LUMIERA_FILE_MASK) < (b->flags&LUMIERA_FILE_MASK)) + return -1; + else if ((a->flags&LUMIERA_FILE_MASK) > (b->flags&LUMIERA_FILE_MASK)) + return 1; + + return 0; +} + + +static void +delete_fn (PSplaynode node) +{ + TODO ("figure name out? or is the handle here always closed"); + lumiera_filedescriptor_delete ((LumieraFiledescriptor) node, NULL); +} + + +static const void* +key_fn (const PSplaynode node) +{ + return node; +} + + + +void +lumiera_filedescriptorregistry_init (void) +{ + TRACE (filedescriptor_dbg); + REQUIRE (!filedescriptorregistry); + + filedescriptorregistry = psplay_new (cmp_fn, key_fn, delete_fn); + if (!filedescriptorregistry) + LUMIERA_DIE (NO_MEMORY); + + lumiera_mutex_init (&filedescriptorregistry_mutex, "filedescriptorregistry", &NOBUG_FLAG (mutex_dbg), NOBUG_CONTEXT); +} + +void +lumiera_filedescriptorregistry_destroy (void) +{ + TRACE (filedescriptor_dbg); + REQUIRE (!psplay_nelements (filedescriptorregistry)); + + lumiera_mutex_destroy (&filedescriptorregistry_mutex, &NOBUG_FLAG (mutex_dbg), NOBUG_CONTEXT); + + if (filedescriptorregistry) + psplay_destroy (filedescriptorregistry); + + filedescriptorregistry = NULL; +} + + +LumieraFiledescriptor +lumiera_filedescriptorregistry_ensure (LumieraFiledescriptor template) +{ + REQUIRE (filedescriptorregistry, "not initialized"); + + LumieraFiledescriptor ret = NULL; + + LUMIERA_MUTEX_SECTION (mutex_sync, &filedescriptorregistry_mutex) + { + /* lookup/create descriptor */ + ret = (LumieraFiledescriptor) psplay_find (filedescriptorregistry, template, 100); + + if (!ret) + { + ret = lumiera_filedescriptor_new (template); + if (!ret) + goto error; + + psplay_insert (filedescriptorregistry, &ret->node, 100); + } + + error: + ; + } + + return ret; +} + + +/** + * Removes a Filedescriptor from the registry. + */ +void +lumiera_filedescriptorregistry_remove (LumieraFiledescriptor self) +{ + LUMIERA_MUTEX_SECTION (mutex_sync, &filedescriptorregistry_mutex) + psplay_remove (filedescriptorregistry, &self->node); +} + + diff --git a/src/backend/filedescriptorregistry.h b/src/backend/filedescriptorregistry.h new file mode 100644 index 000000000..c3a03da83 --- /dev/null +++ b/src/backend/filedescriptorregistry.h @@ -0,0 +1,58 @@ +/* + filedescriptorregistry.h - register all files in use + + Copyright (C) Lumiera.org + 2008, 2010, Christian Thaeter + + This program is free software; you can redistribute it and/or + modify it under the terms of the GNU General Public License as + published by the Free Software Foundation; either version 2 of the + License, or (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, write to the Free Software + Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA. +*/ + +#ifndef LUMIERA_FILEDESCRIPTORREGISTRY_H +#define LUMIERA_FILEDESCRIPTORREGISTRY_H + +#include "backend/filedescriptor.h" + + +/** + * Initialize the global filedescriptor registry. + * Opening hardlinked files will be targeted to the same filedescriptor. + * This function never fails but dies on error. TODO backend/subsystem failure + */ +void +lumiera_filedescriptorregistry_init (void); + +/** + * Destroy and free the global filedescriptor registry. + * Never fails. + */ +void +lumiera_filedescriptorregistry_destroy (void); + +/** + * Ensures that a filedescriptor is in the registry. + * Looks template up and if not found, create a new one from template. + * @return filedescriptor from registry + */ +LumieraFiledescriptor +lumiera_filedescriptorregistry_ensure (LumieraFiledescriptor template); + +/** + * Removes a Filedescriptor from the registry. + */ +void +lumiera_filedescriptorregistry_remove (LumieraFiledescriptor self); + + +#endif diff --git a/src/backend/mmap.c b/src/backend/mmap.c index 4a69144f4..be3585872 100644 --- a/src/backend/mmap.c +++ b/src/backend/mmap.c @@ -59,21 +59,16 @@ lumiera_mmap_init (LumieraMMap self, LumieraFile file, off_t start, size_t size) REQUIRE (start >= 0); REQUIRE (size); - static int once = 0; - if (!once) - { - once = 1; - /** - * default size for the mmapping window - * 128MB on 32 bit arch - * 2GB on 64 bit arch - */ + /** + * default size for the mmapping window + * 128MB on 32 bit arch + * 2GB on 64 bit arch + */ #if SIZE_MAX <= 4294967295U - lumiera_config_setdefault ("backend.mmap.window_size = 134217728"); + lumiera_config_setdefault ("backend.mmap.window_size = 134217728"); #else - lumiera_config_setdefault ("backend.mmap.window_size = 2147483648"); + lumiera_config_setdefault ("backend.mmap.window_size = 2147483648"); #endif - } long long mmap_window_size = 0; lumiera_config_number_get ("backend.mmap.window_size", &mmap_window_size); @@ -207,7 +202,6 @@ lumiera_mmap_init (LumieraMMap self, LumieraFile file, off_t start, size_t size) etruncate: efile: lumiera_file_handle_release (file); - lumiera_free (self); return NULL; } diff --git a/src/lib/resourcecollector.c b/src/backend/resourcecollector.c similarity index 94% rename from src/lib/resourcecollector.c rename to src/backend/resourcecollector.c index 84af12f86..7ef6a48de 100644 --- a/src/lib/resourcecollector.c +++ b/src/backend/resourcecollector.c @@ -32,7 +32,6 @@ llist lumiera_resourcecollector_registry[LUMIERA_RESOURCE_END]; lumiera_mutex lumiera_resourcecollector_lock; -static pthread_once_t lumiera_resourcecollector_once = PTHREAD_ONCE_INIT; struct lumiera_resourcehandler_struct { @@ -42,8 +41,8 @@ struct lumiera_resourcehandler_struct }; -static void -lumiera_resourcecollector_init_ (void) +void +lumiera_resourcecollector_init (void) { //NOBUG_INIT_FLAG (resourcecollector); TRACE (resourcecollector_dbg); @@ -74,8 +73,6 @@ lumiera_resourcecollector_run (enum lumiera_resource which, enum lumiera_resourc { TRACE (resourcecollector_dbg); - pthread_once (&lumiera_resourcecollector_once, lumiera_resourcecollector_init_); - LUMIERA_MUTEX_SECTION (mutex_sync, &lumiera_resourcecollector_lock) { for (enum lumiera_resource_try progress = LUMIERA_RESOURCE_NONE; progress < *iteration; ++*iteration) @@ -119,8 +116,6 @@ lumiera_resourcecollector_run (enum lumiera_resource which, enum lumiera_resourc LumieraResourcehandler lumiera_resourcecollector_register_handler (enum lumiera_resource resource, lumiera_resource_handler_fn handler, void* data) { - pthread_once (&lumiera_resourcecollector_once, lumiera_resourcecollector_init_); - TRACE (resourcecollector_dbg); LumieraResourcehandler self = lumiera_malloc (sizeof (*self)); diff --git a/src/lib/resourcecollector.h b/src/backend/resourcecollector.h similarity index 95% rename from src/lib/resourcecollector.h rename to src/backend/resourcecollector.h index e7addae49..1d41e3e44 100644 --- a/src/lib/resourcecollector.h +++ b/src/backend/resourcecollector.h @@ -93,15 +93,23 @@ enum lumiera_resource_try * @return indication what the the handler really did (LUMIERA_RESOURCE_NONE when it didn't obey the request) */ typedef enum lumiera_resource_try (*lumiera_resource_handler_fn)(enum lumiera_resource_try itr, void* data, void* context); +typedef int (*lumiera_resourcecollector_run_fn) (enum lumiera_resource which, enum lumiera_resource_try* iteration, void* context); typedef struct lumiera_resourcehandler_struct lumiera_resourcehandler; typedef lumiera_resourcehandler* LumieraResourcehandler; +/** + * Initialize the Resourcecollector. + * The resourcecollector is singleton and can be used after initialized once. + */ +void +lumiera_resourcecollector_init (void); + + /** * Destroy the resource collector registry. * Unregisters and deletes all handlers. - * Note that there is no resourcecollector_init() function, initialization is automatic on first use. */ void lumiera_resourcecollector_destroy (void); diff --git a/src/lib/Makefile.am b/src/lib/Makefile.am index 7b6218ad8..9cc532682 100644 --- a/src/lib/Makefile.am +++ b/src/lib/Makefile.am @@ -47,7 +47,6 @@ liblumiera_la_SOURCES = \ $(liblumiera_la_srcdir)/psplay.c \ $(liblumiera_la_srcdir)/mrucache.c \ $(liblumiera_la_srcdir)/time.c \ - $(liblumiera_la_srcdir)/resourcecollector.c \ $(liblumiera_la_srcdir)/allocationcluster.cpp \ $(liblumiera_la_srcdir)/lumitime.cpp \ $(liblumiera_la_srcdir)/lifecycle.cpp \ @@ -77,7 +76,6 @@ noinst_HEADERS += \ $(liblumiera_la_srcdir)/psplay.h \ $(liblumiera_la_srcdir)/mrucache.h \ $(liblumiera_la_srcdir)/time.h \ - $(liblumiera_la_srcdir)/resourcecollector.h \ $(liblumiera_la_srcdir)/ppmpl.h \ $(liblumiera_la_srcdir)/allocationcluster.hpp \ $(liblumiera_la_srcdir)/scoped-holder.hpp \ diff --git a/src/lib/error.c b/src/lib/error.c index 161b695e6..0de5f4b7a 100644 --- a/src/lib/error.c +++ b/src/lib/error.c @@ -58,10 +58,8 @@ static void lumiera_error_tls_delete (void* err) { if (err) - { - free (((LumieraErrorcontext)err)->extra); - free (err); - } + free (((LumieraErrorcontext)err)->extra); + free(err); } static void diff --git a/src/lib/mpool.c b/src/lib/mpool.c index 8c1d88a93..1c77aadcd 100644 --- a/src/lib/mpool.c +++ b/src/lib/mpool.c @@ -37,10 +37,20 @@ #define MPOOL_C(c) c ## UL #endif +/* + defaults for the hooks, used when creating mpools + */ +void *(*mpool_malloc_hook)(size_t size) = malloc; +void (*mpool_free_hook)(void *ptr) = free; + +/** called after a mpool got initialized */ +void (*mpool_init_hook) (MPool self) = NULL; +/** called before a mpool gets destroyed */ +void (*mpool_destroy_hook) (MPool self) = NULL; + /* Cluster and node structures are private */ - typedef struct mpoolcluster_struct mpoolcluster; typedef mpoolcluster* MPoolcluster; typedef const mpoolcluster* const_MPoolcluster; @@ -94,6 +104,12 @@ mpool_init (MPool self, size_t elem_size, unsigned elements_per_cluster, mpool_d self->elements_free = 0; self->destroy = dtor; self->locality = NULL; + + self->malloc_hook = mpool_malloc_hook; + self->free_hook = mpool_free_hook; + + if (mpool_init_hook) + mpool_init_hook (self); } return self; @@ -129,6 +145,9 @@ mpool_destroy (MPool self) TRACE (mpool_dbg, "%p", self); if (self) { + if (mpool_destroy_hook) + mpool_destroy_hook (self); + LLIST_WHILE_TAIL(&self->clusters, cluster) { if (self->destroy) @@ -144,7 +163,7 @@ mpool_destroy (MPool self) llist_unlink_fast_ (cluster); TRACE (mpool_dbg, "freeing cluster %p" , cluster); - free (cluster); + self->free_hook (cluster); } llist_init (&self->freelist); @@ -156,11 +175,19 @@ mpool_destroy (MPool self) } +MPool +mpool_purge (MPool self) +{ + // not UNIMPLEMENTED because valid no-op + PLANNED("To be implemented"); + return self; +} + MPool mpool_cluster_alloc_ (MPool self) { - MPoolcluster cluster = malloc (self->cluster_size); + MPoolcluster cluster = self->malloc_hook (self->cluster_size); TRACE (mpool_dbg, "%p", cluster); if (!cluster) diff --git a/src/lib/mpool.h b/src/lib/mpool.h index aa9b66562..3cc9f4953 100644 --- a/src/lib/mpool.h +++ b/src/lib/mpool.h @@ -77,9 +77,20 @@ struct mpool_struct unsigned elements_free; /* a counter of free elements is the price we pay to support a reserve() operation */ void* locality; mpool_destroy_fn destroy; + void *(*malloc_hook)(size_t); + void (*free_hook)(void *); + void* udata; /* free to use by the user, resourcecollector stuff in lumiera*/ }; +extern void *(*mpool_malloc_hook)(size_t size); +extern void (*mpool_free_hook)(void *ptr); + +/** called after a mpool got initialized */ +extern void (*mpool_init_hook) (MPool self); +/** called before a mpool gets destroyed */ +extern void (*mpool_destroy_hook) (MPool self); + /* //index.mpool_init xref:mpool_init[mpool_init()]:: initialize a new memory pool //mpool [[mpool_init]] @@ -128,6 +139,18 @@ mpool_init (MPool self, size_t elem_size, unsigned elements_per_cluster, mpool_d MPool mpool_destroy (MPool self); +/* +//index.mpool_purge xref:mpool_purge[mpool_purge()]:: free unused clusters +//mpool [[mpool_purge]] +//mpool .mpool_purge +//mpool +//mpool TODO +//mpool +//mpool +*/ +MPool +mpool_purge (MPool self); + /* //index.mpool_available xref:mpool_available[mpool_available()]:: query number of free elements diff --git a/src/lib/safeclib.c b/src/lib/safeclib.c index b6503a998..989648812 100644 --- a/src/lib/safeclib.c +++ b/src/lib/safeclib.c @@ -21,6 +21,8 @@ #include "lib/error.h" #include "lib/safeclib.h" +#include "backend/resourcecollector.h" + #include #include #include @@ -31,12 +33,40 @@ LUMIERA_ERROR_DEFINE (NO_MEMORY, "Out of Memory!"); + + +/* placeholder function until the resourcecollector gets hooked in */ +static int +die_no_mem (enum lumiera_resource which, enum lumiera_resource_try* iteration, void* context) +{ + (void) which; (void) iteration; (void) context; + LUMIERA_DIE (NO_MEMORY); + return 0; /* not reached */ +} + +static lumiera_resourcecollector_run_fn lumiera_safeclib_resourcecollector_run_hook = die_no_mem; + +void +lumiera_safeclib_set_resourcecollector (void* hook) +{ + if (hook) + lumiera_safeclib_resourcecollector_run_hook = (lumiera_resourcecollector_run_fn)hook; + else + lumiera_safeclib_resourcecollector_run_hook = die_no_mem; +} + + void* lumiera_malloc (size_t size) { - void* o = size ? malloc (size) : NULL; - if (!o) - LUMIERA_DIE (NO_MEMORY); + enum lumiera_resource_try iteration = LUMIERA_RESOURCE_ONE; + void* o = NULL; + + if (size) + do + o = malloc (size); + while (!o && lumiera_safeclib_resourcecollector_run_hook (LUMIERA_RESOURCE_MEMORY, &iteration, &size)); + return o; } @@ -44,9 +74,16 @@ lumiera_malloc (size_t size) void* lumiera_calloc (size_t n, size_t size) { - void* o = (n&&size)? calloc (n, size) : NULL; - if (!o) - LUMIERA_DIE (NO_MEMORY); + enum lumiera_resource_try iteration = LUMIERA_RESOURCE_ONE; + void* o = NULL; + + size_t gross = n*size; + + if (n&&size) + do + o = calloc (n, size); + while (!o && lumiera_safeclib_resourcecollector_run_hook (LUMIERA_RESOURCE_MEMORY, &iteration, &gross)); + return o; } @@ -54,9 +91,13 @@ lumiera_calloc (size_t n, size_t size) void* lumiera_realloc (void* ptr, size_t size) { - void* o = size ? realloc (ptr, size) : NULL; - if (!o) - LUMIERA_DIE (NO_MEMORY); + enum lumiera_resource_try iteration = LUMIERA_RESOURCE_ONE; + void* o = NULL; + + if (size) + do + o = realloc (ptr, size); + while (!o && lumiera_safeclib_resourcecollector_run_hook (LUMIERA_RESOURCE_MEMORY, &iteration, &size)); return o; } @@ -65,14 +106,16 @@ lumiera_realloc (void* ptr, size_t size) char* lumiera_strndup (const char* str, size_t len) { - char* o; - if (str) - o = strndup (str, len); - else - o = strdup (""); + enum lumiera_resource_try iteration = LUMIERA_RESOURCE_ONE; + void* o = NULL; + + do + if (str && len) + o = strndup (str, len); + else + o = strdup (""); + while (!o && lumiera_safeclib_resourcecollector_run_hook (LUMIERA_RESOURCE_MEMORY, &iteration, &len)); - if (!o) - LUMIERA_DIE (NO_MEMORY); return o; } @@ -234,3 +277,12 @@ lumiera_tmpbuf_tr (const char* in, const char* from, const char* to, const char* return ret; } + + +/* +// Local Variables: +// mode: C +// c-file-style: "gnu" +// indent-tabs-mode: nil +// End: +*/ diff --git a/src/lib/safeclib.h b/src/lib/safeclib.h index ed0d73fd3..be322aa0b 100644 --- a/src/lib/safeclib.h +++ b/src/lib/safeclib.h @@ -29,6 +29,18 @@ */ LUMIERA_ERROR_DECLARE(NO_MEMORY); +/** + * Install the resourcecollector run hook. + * The resourcecollectr must be hooked into the safeclib at bootup after it got + * initialized and removed from it before shut down. Without resourcecollector + * failed allocations will abort(). + * @param hook pointer to the resourcecollector_run function, must be of type + * lumiera_resourcecollector_run_fn but we dont want a dependency on backend in this header + */ +void +lumiera_safeclib_set_resourcecollector (void* hook); + + /** * Allocate memory. * always succeeds or dies diff --git a/tests/30mmap.tests b/tests/30mmap.tests index 66bc7b69a..8bb70dd15 100644 --- a/tests/30mmap.tests +++ b/tests/30mmap.tests @@ -10,27 +10,30 @@ TEST "mmap not released should fail" mmap_forget_releasing </dev/null TEST "basic mmap" mmap_simple </dev/null TEST "use mmap twice" mmap_checkout_twice </dev/null TEST "reuse mmap" mmap_checkout_again </dev/null diff --git a/tests/20resourcecollector.tests b/tests/30resourcecollector.tests similarity index 100% rename from tests/20resourcecollector.tests rename to tests/30resourcecollector.tests diff --git a/tests/Makefile.am b/tests/Makefile.am index d00d2652e..f14444cd3 100644 --- a/tests/Makefile.am +++ b/tests/Makefile.am @@ -73,9 +73,10 @@ test_filemmap_LDADD = $(LUMIERA_LIBS) liblumierabackend.la liblumieracommon.la l # FIXME stray dependencies on proc check_PROGRAMS += test-resourcecollector -test_resourcecollector_SOURCES = $(tests_srcdir)/library/test-resourcecollector.c +test_resourcecollector_SOURCES = $(tests_srcdir)/backend/test-resourcecollector.c test_resourcecollector_CPPFLAGS = $(AM_CPPFLAGS) -std=gnu99 -Wall -Werror -I$(top_srcdir)/src/ -test_resourcecollector_LDADD = $(LUMIERA_LIBS) liblumiera.la +test_resourcecollector_LDADD = $(LUMIERA_LIBS) liblumiera.la liblumierabackend.la liblumieraproc.la liblumieracommon.la +# FIXME stray dependencies on proc check_PROGRAMS += test-slist test_slist_SOURCES = $(tests_srcdir)/library/test-slist.c diff --git a/tests/library/test-resourcecollector.c b/tests/backend/test-resourcecollector.c similarity index 94% rename from tests/library/test-resourcecollector.c rename to tests/backend/test-resourcecollector.c index d3e6d8483..166dc1129 100644 --- a/tests/library/test-resourcecollector.c +++ b/tests/backend/test-resourcecollector.c @@ -19,7 +19,7 @@ Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA. */ -#include "lib/resourcecollector.h" +#include "backend/resourcecollector.h" #include "tests/test.h" @@ -50,6 +50,7 @@ TESTS_BEGIN TEST ("basic") { + lumiera_resourcecollector_init (); lumiera_resourcecollector_register_handler (LUMIERA_RESOURCE_MEMORY, test_memory_handler, NULL); lumiera_resourcecollector_destroy (); } @@ -57,6 +58,7 @@ TEST ("basic") TEST ("memory_collection_mockup") { REQUIRE (argv[2]); + lumiera_resourcecollector_init (); lumiera_resourcecollector_register_handler (LUMIERA_RESOURCE_MEMORY, test_memory_handler, NULL);