benn/lumiera_
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Merge remote branch 'ct/backend_devel' into second-tp-attempt

Browse source 
Conflicts:
	src/lib/error.c
This commit is contained in:
Michael Ploujnikov 2010-02-09 07:29:43 -05:00
commit 5606eb2d83
22 changed files with 524 additions and 188 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

6
src/backend/Makefile.am
View file

@ -30,10 +30,12 @@ liblumierabackend_la_SOURCES = \
$(liblumierabackend_la_srcdir)/file.c \
$(liblumierabackend_la_srcdir)/filehandle.c \
$(liblumierabackend_la_srcdir)/filedescriptor.c \
$(liblumierabackend_la_srcdir)/filedescriptorregistry.c \
$(liblumierabackend_la_srcdir)/filehandlecache.c \
$(liblumierabackend_la_srcdir)/mmap.c \
$(liblumierabackend_la_srcdir)/mmapings.c \
$(liblumierabackend_la_srcdir)/mmapcache.c \
$(liblumierabackend_la_srcdir)/resourcecollector.c \
$(liblumierabackend_la_srcdir)/threadpool-init.cpp \
$(liblumierabackend_la_srcdir)/enginefacade.cpp \
$(liblumierabackend_la_srcdir)/scriptrunnerfacade.cpp \
@ -47,8 +49,10 @@ noinst_HEADERS += \
$(liblumierabackend_la_srcdir)/file.h \
$(liblumierabackend_la_srcdir)/filehandle.h \
$(liblumierabackend_la_srcdir)/filedescriptor.h \
$(liblumierabackend_la_srcdir)/filedescriptorregistry.h \
$(liblumierabackend_la_srcdir)/filehandlecache.h \
$(liblumierabackend_la_srcdir)/mmap.h \
$(liblumierabackend_la_srcdir)/mmapings.h \
$(liblumierabackend_la_srcdir)/mmapcache.h
$(liblumierabackend_la_srcdir)/mmapcache.h \
$(liblumierabackend_la_srcdir)/resourcecollector.h

81
src/backend/backend.c
View file

@ -21,13 +21,16 @@
#include "include/logging.h"
#include "lib/safeclib.h"
#include "lib/mpool.h"
#include "backend/backend.h"
#include "common/config.h"
#include "backend/filehandlecache.h"
#include "backend/filedescriptor.h"
#include "backend/filedescriptorregistry.h"
#include "backend/mmapcache.h"
#include "backend/threadpool.h"
#include "backend/resourcecollector.h"
#include <unistd.h>
#include <sys/resource.h>
@ -49,8 +52,20 @@
//NOBUG_DECLARE_FLAG (mmapcache);
static enum lumiera_resource_try
lumiera_backend_mpool_purge (enum lumiera_resource_try itr, void* data, void* context);
static void
lumiera_backend_resourcecollector_register_mpool (MPool self);
static void
lumiera_backend_resourcecollector_unregister_mpool (MPool self);
size_t lumiera_backend_pagesize;
int
lumiera_backend_init (void)
{
@ -64,16 +79,30 @@ lumiera_backend_init (void)
//NOBUG_INIT_FLAG (mmapcache);
TRACE (backend_dbg);
lumiera_mutex_init (&lumiera_filecreate_mutex, "fileaccess", &NOBUG_FLAG (mutex_dbg), NOBUG_CONTEXT);
lumiera_resourcecollector_init ();
/* hook the resourcecollector into the mpool*/
mpool_malloc_hook = lumiera_malloc;
mpool_free_hook = lumiera_free;
mpool_init_hook = lumiera_backend_resourcecollector_register_mpool;
mpool_destroy_hook = lumiera_backend_resourcecollector_unregister_mpool;
/* hook the resourcecollector into the safeclib allocation functions */
lumiera_safeclib_set_resourcecollector (lumiera_resourcecollector_run);
PLANNED("The resourcecollector aborts by default when there is no final strategy for recovery, TODO: initiate sane shutdown");
lumiera_threadpool_init ();
PLANNED ("hook threadpool into the resourcecollector (maybe in threadpool_init() instead here");
lumiera_filedescriptor_registry_init ();
lumiera_filedescriptorregistry_init ();
lumiera_backend_pagesize = sysconf(_SC_PAGESIZE);
TODO ("add config options to override following defaults");
const char* filehandles = lumiera_tmpbuf_snprintf (SIZE_MAX,
"backend.file.max_handles = %d",
/* roughly 2/3 of all availables filehandles are managed by the backend */
@ -113,8 +142,54 @@ void
lumiera_backend_destroy (void)
{
TRACE (backend_dbg);
lumiera_mmapcache_delete ();
lumiera_filehandlecache_delete ();
lumiera_filedescriptor_registry_destroy ();
lumiera_filedescriptorregistry_destroy ();
lumiera_threadpool_destroy ();
lumiera_safeclib_set_resourcecollector (NULL);
mpool_init_hook = NULL;
mpool_destroy_hook = NULL;
mpool_malloc_hook = malloc;
mpool_free_hook = free;
lumiera_resourcecollector_destroy ();
lumiera_mutex_destroy (&lumiera_filecreate_mutex, &NOBUG_FLAG (mutex_dbg), NOBUG_CONTEXT);
}
static enum lumiera_resource_try
lumiera_backend_mpool_purge (enum lumiera_resource_try itr, void* data, void* context)
{
(void) context;
(void) data;
(void) itr;
TODO("mpool_purge ((MPool) data)");
return LUMIERA_RESOURCE_NONE;
}
static void
lumiera_backend_resourcecollector_register_mpool (MPool self)
{
self->udata =
lumiera_resourcecollector_register_handler (LUMIERA_RESOURCE_MEMORY, lumiera_backend_mpool_purge, self);
}
static void
lumiera_backend_resourcecollector_unregister_mpool (MPool self)
{
lumiera_resourcehandler_unregister ((LumieraResourcehandler) self->udata);
}
/*
// Local Variables:
// mode: C
// c-file-style: "gnu"
// indent-tabs-mode: nil
// End:
*/

9
src/backend/backend.h
View file

@ -21,12 +21,21 @@
#ifndef LUMIERA_BACKEND_H
#define LUMIERA_BACKEND_H
#include "lib/mutex.h"
#include <nobug.h>
//NOBUG_DECLARE_FLAG (backend);
extern size_t lumiera_backend_pagesize;
/**
* Protect lookup and creation of files.
* Trying to access a nonexistent file with O_CREAT would be racy.
* Defined in filedescriptor.c
*/
extern lumiera_mutex lumiera_filecreate_mutex;
int
lumiera_backend_init (void);

22
src/backend/file.c
View file

@ -54,11 +54,14 @@ lumiera_file_init (LumieraFile self, const char* name, int flags)
}
LumieraFile
lumiera_file_destroy (LumieraFile self)
lumiera_file_destroy (LumieraFile self, int do_unlink)
{
TRACE (file_dbg);
lumiera_filedescriptor_release (self->descriptor, self->name, &self->node);
if (do_unlink)
unlink (self->name);
lumiera_free (self->name);
return self;
}
@ -85,7 +88,16 @@ lumiera_file_delete (LumieraFile self)
{
TRACE (file_dbg);
TRACE (file, "close file '%s'", self->name);
lumiera_free (lumiera_file_destroy (self));
lumiera_free (lumiera_file_destroy (self, 0));
}
void
lumiera_file_delete_unlink (LumieraFile self)
{
TRACE (file_dbg);
TRACE (file, "close and unlink file '%s'", self->name);
lumiera_free (lumiera_file_destroy (self, 1));
}
@ -124,6 +136,12 @@ lumiera_file_chunksize_set (LumieraFile self, size_t chunksize)
size_t
lumiera_file_chunksize_get (LumieraFile self)
{
if (!self->descriptor->mmapings)
{
LUMIERA_ERROR_SET (file, FILE_NOCHUNKSIZE, lumiera_filedescriptor_name (self->descriptor));
return 0;
}
return self->descriptor->mmapings->chunksize;
}

13
src/backend/file.h
View file

@ -91,12 +91,11 @@ lumiera_file_init (LumieraFile self, const char* name, int flags);
* Destroy a file structure.
* frees all associated resources, releases the filedescriptor etc.
* @param self file structure to be destroyed
* @param chunksize allocation/mmaping granularity, must be 2's exponent of pagesize
* only used at the first access to a file and ignored for subsequnet accesses
* @param do_unlink if 1 then delete the file physically from disk (only the associated name)
* @return self
*/
LumieraFile
lumiera_file_destroy (LumieraFile self);
lumiera_file_destroy (LumieraFile self, int do_unlink);
/**
@ -117,6 +116,14 @@ void
lumiera_file_delete (LumieraFile self);
/**
* Frees a file structure and deletes the associated file name from disk.
* @param self file structure to be freed
*/
void
lumiera_file_delete_unlink (LumieraFile self);
/**
* Get a POSIX filehandle for a file.
* Filehandles are opened on demand and must be acquired for use.

142
src/backend/filedescriptor.c
View file

@ -25,6 +25,7 @@
#include "backend/file.h"
#include "backend/filedescriptor.h"
#include "backend/filedescriptorregistry.h"
#include "backend/filehandle.h"
#include "backend/filehandlecache.h"
@ -37,94 +38,19 @@
//NOBUG_DEFINE_FLAG_PARENT (filedescriptor, file_all);
/*
Filedescriptor registry
This registry stores all acquired filedescriptors for lookup, they will be freed when not referenced anymore.
*/
static PSplay registry = NULL;
static lumiera_mutex registry_mutex;
static int
cmp_fn (const void* keya, const void* keyb)
{
const LumieraFiledescriptor a = (const LumieraFiledescriptor)keya;
const LumieraFiledescriptor b = (const LumieraFiledescriptor)keyb;
if (a->stat.st_dev < b->stat.st_dev)
return -1;
else if (a->stat.st_dev > b->stat.st_dev)
return 1;
if (a->stat.st_ino < b->stat.st_ino)
return -1;
else if (a->stat.st_ino > b->stat.st_ino)
return 1;
if ((a->flags&LUMIERA_FILE_MASK) < (b->flags&LUMIERA_FILE_MASK))
return -1;
else if ((a->flags&LUMIERA_FILE_MASK) > (b->flags&LUMIERA_FILE_MASK))
return 1;
return 0;
}
static void
delete_fn (PSplaynode node)
{
TODO ("figure name out? or is the handle here always closed");
lumiera_filedescriptor_delete ((LumieraFiledescriptor) node, NULL);
}
static const void*
key_fn (const PSplaynode node)
{
return node;
}
void
lumiera_filedescriptor_registry_init (void)
{
//NOBUG_INIT_FLAG (filedescriptor);
TRACE (filedescriptor_dbg);
REQUIRE (!registry);
registry = psplay_new (cmp_fn, key_fn, delete_fn);
if (!registry)
LUMIERA_DIE (NO_MEMORY);
lumiera_mutex_init (&registry_mutex, "filedescriptor_registry", &NOBUG_FLAG (mutex_dbg), NOBUG_CONTEXT);
}
void
lumiera_filedescriptor_registry_destroy (void)
{
TRACE (filedescriptor_dbg);
REQUIRE (!psplay_nelements (registry));
lumiera_mutex_destroy (&registry_mutex, &NOBUG_FLAG (mutex_dbg), NOBUG_CONTEXT);
if (registry)
psplay_destroy (registry);
registry = NULL;
}
/* lookup and creation of files, initialized in backend.c */
lumiera_mutex lumiera_filecreate_mutex;
LumieraFiledescriptor
lumiera_filedescriptor_acquire (const char* name, int flags, LList filenode)
{
TRACE (filedescriptor_dbg, "%s", name);
REQUIRE (registry, "not initialized");
REQUIRE (llist_is_empty (filenode));
LumieraFiledescriptor dest = NULL;
LUMIERA_MUTEX_SECTION (mutex_sync, &registry_mutex)
LUMIERA_MUTEX_SECTION (mutex_sync, &lumiera_filecreate_mutex)
{
lumiera_filedescriptor fdesc;
fdesc.flags = flags;
@ -167,25 +93,21 @@ lumiera_filedescriptor_acquire (const char* name, int flags, LList filenode)
}
/* lookup/create descriptor */
dest = (LumieraFiledescriptor) psplay_find (registry, &fdesc, 100);
dest = lumiera_filedescriptorregistry_ensure (&fdesc);
if (!dest)
{
dest = lumiera_filedescriptor_new (&fdesc);
if (!dest)
goto error;
if (dest)
llist_insert_head (&dest->files, filenode);
psplay_insert (registry, &dest->node, 100);
}
llist_insert_head (&dest->files, filenode);
error: ;
error:
;
}
return dest;
}
void
lumiera_filedescriptor_release (LumieraFiledescriptor self, const char* name, LList filenode)
{
@ -288,29 +210,25 @@ lumiera_filedescriptor_delete (LumieraFiledescriptor self, const char* name)
{
TRACE (filedescriptor_dbg, "%p %s", self, name);
LUMIERA_MUTEX_SECTION (mutex_sync, &registry_mutex)
REQUIRE (llist_is_empty (&self->files));
lumiera_filedescriptorregistry_remove (self);
TODO ("destruct other members (WIP)");
lumiera_mmapings_delete (self->mmapings);
if (self->handle && name && ((self->flags & O_RDWR) == O_RDWR))
{
REQUIRE (llist_is_empty (&self->files));
psplay_remove (registry, &self->node);
TODO ("destruct other members (WIP)");
if (self->handle && name && ((self->flags & O_RDWR) == O_RDWR))
{
TRACE (filedescriptor_dbg, "truncate %s to %lld", name, (long long)self->realsize);
lumiera_filehandlecache_checkout (self->handle);
int dummy = ftruncate (lumiera_filehandle_handle (self->handle), self->realsize);
(void) dummy; /* this is present to silence a warning */
TODO ("handle error case better");
lumiera_filehandlecache_checkin (self->handle);
}
lumiera_mmapings_delete (self->mmapings);
TODO ("release filehandle");
lumiera_mutex_destroy (&self->lock, &NOBUG_FLAG (mutex_dbg), NOBUG_CONTEXT);
lumiera_free (self);
TRACE (filedescriptor_dbg, "truncate %s to %lld", name, (long long)self->realsize);
lumiera_filehandlecache_checkout (self->handle);
int dummy = ftruncate (lumiera_filehandle_handle (self->handle), self->realsize);
(void) dummy; /* this is present to silence a warning */
TODO ("handle error case better");
lumiera_filehandlecache_checkin (self->handle);
TODO ("really release filehandle");
}
lumiera_mutex_destroy (&self->lock, &NOBUG_FLAG (mutex_dbg), NOBUG_CONTEXT);
lumiera_free (self);
}

16
src/backend/filedescriptor.h
View file

@ -75,22 +75,6 @@ struct lumiera_filedescriptor_struct
llist files;
};
/**
* Initialize the global filedescriptor registry.
* All filedescriptors are looked up by dev/ino/open-flags in a hashtable. Opening hardlinked
* files will be targeted to the same filedescriptor.
* This function never fails but dies on error.
*/
void
lumiera_filedescriptor_registry_init (void);
/**
* Destroy and free the global filedescriptor registry.
* Never fails.
*/
void
lumiera_filedescriptor_registry_destroy (void);
/**
* Find existing filedescriptor or create one

150
src/backend/filedescriptorregistry.c Normal file
View file

@ -0,0 +1,150 @@
/*
filedescriptorregistry.c - register all files in use
Copyright (C) Lumiera.org
2008, 2010, Christian Thaeter <ct@pipapo.org>
This program is free software; you can redistribute it and/or
modify it under the terms of the GNU General Public License as
published by the Free Software Foundation; either version 2 of the
License, or (at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program; if not, write to the Free Software
Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
*/
#include "include/logging.h"
#include "lib/safeclib.h"
#include "lib/mutex.h"
#include "lib/psplay.h"
#include "backend/file.h"
#include "backend/filedescriptor.h"
#include "backend/filedescriptorregistry.h"
//NOBUG_DEFINE_FLAG_PARENT (filedescriptorregistry, file_all);
/*
Filedescriptor registry
This registry stores all acquired filedescriptors for lookup, they will be freed when not referenced anymore.
*/
static PSplay filedescriptorregistry = NULL;
static lumiera_mutex filedescriptorregistry_mutex;
static int
cmp_fn (const void* keya, const void* keyb)
{
const LumieraFiledescriptor a = (const LumieraFiledescriptor)keya;
const LumieraFiledescriptor b = (const LumieraFiledescriptor)keyb;
if (a->stat.st_dev < b->stat.st_dev)
return -1;
else if (a->stat.st_dev > b->stat.st_dev)
return 1;
if (a->stat.st_ino < b->stat.st_ino)
return -1;
else if (a->stat.st_ino > b->stat.st_ino)
return 1;
if ((a->flags&LUMIERA_FILE_MASK) < (b->flags&LUMIERA_FILE_MASK))
return -1;
else if ((a->flags&LUMIERA_FILE_MASK) > (b->flags&LUMIERA_FILE_MASK))
return 1;
return 0;
}
static void
delete_fn (PSplaynode node)
{
TODO ("figure name out? or is the handle here always closed");
lumiera_filedescriptor_delete ((LumieraFiledescriptor) node, NULL);
}
static const void*
key_fn (const PSplaynode node)
{
return node;
}
void
lumiera_filedescriptorregistry_init (void)
{
TRACE (filedescriptor_dbg);
REQUIRE (!filedescriptorregistry);
filedescriptorregistry = psplay_new (cmp_fn, key_fn, delete_fn);
if (!filedescriptorregistry)
LUMIERA_DIE (NO_MEMORY);
lumiera_mutex_init (&filedescriptorregistry_mutex, "filedescriptorregistry", &NOBUG_FLAG (mutex_dbg), NOBUG_CONTEXT);
}
void
lumiera_filedescriptorregistry_destroy (void)
{
TRACE (filedescriptor_dbg);
REQUIRE (!psplay_nelements (filedescriptorregistry));
lumiera_mutex_destroy (&filedescriptorregistry_mutex, &NOBUG_FLAG (mutex_dbg), NOBUG_CONTEXT);
if (filedescriptorregistry)
psplay_destroy (filedescriptorregistry);
filedescriptorregistry = NULL;
}
LumieraFiledescriptor
lumiera_filedescriptorregistry_ensure (LumieraFiledescriptor template)
{
REQUIRE (filedescriptorregistry, "not initialized");
LumieraFiledescriptor ret = NULL;
LUMIERA_MUTEX_SECTION (mutex_sync, &filedescriptorregistry_mutex)
{
/* lookup/create descriptor */
ret = (LumieraFiledescriptor) psplay_find (filedescriptorregistry, template, 100);
if (!ret)
{
ret = lumiera_filedescriptor_new (template);
if (!ret)
goto error;
psplay_insert (filedescriptorregistry, &ret->node, 100);
}
error:
;
}
return ret;
}
/**
* Removes a Filedescriptor from the registry.
*/
void
lumiera_filedescriptorregistry_remove (LumieraFiledescriptor self)
{
LUMIERA_MUTEX_SECTION (mutex_sync, &filedescriptorregistry_mutex)
psplay_remove (filedescriptorregistry, &self->node);
}

58
src/backend/filedescriptorregistry.h Normal file
View file

@ -0,0 +1,58 @@
/*
filedescriptorregistry.h - register all files in use
Copyright (C) Lumiera.org
2008, 2010, Christian Thaeter <ct@pipapo.org>
This program is free software; you can redistribute it and/or
modify it under the terms of the GNU General Public License as
published by the Free Software Foundation; either version 2 of the
License, or (at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program; if not, write to the Free Software
Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
*/
#ifndef LUMIERA_FILEDESCRIPTORREGISTRY_H
#define LUMIERA_FILEDESCRIPTORREGISTRY_H
#include "backend/filedescriptor.h"
/**
* Initialize the global filedescriptor registry.
* Opening hardlinked files will be targeted to the same filedescriptor.
* This function never fails but dies on error. TODO backend/subsystem failure
*/
void
lumiera_filedescriptorregistry_init (void);
/**
* Destroy and free the global filedescriptor registry.
* Never fails.
*/
void
lumiera_filedescriptorregistry_destroy (void);
/**
* Ensures that a filedescriptor is in the registry.
* Looks template up and if not found, create a new one from template.
* @return filedescriptor from registry
*/
LumieraFiledescriptor
lumiera_filedescriptorregistry_ensure (LumieraFiledescriptor template);
/**
* Removes a Filedescriptor from the registry.
*/
void
lumiera_filedescriptorregistry_remove (LumieraFiledescriptor self);
#endif

20
src/backend/mmap.c
View file

@ -59,21 +59,16 @@ lumiera_mmap_init (LumieraMMap self, LumieraFile file, off_t start, size_t size)
REQUIRE (start >= 0);
REQUIRE (size);
static int once = 0;
if (!once)
{
once = 1;
/**
* default size for the mmapping window
* 128MB on 32 bit arch
* 2GB on 64 bit arch
*/
/**
* default size for the mmapping window
* 128MB on 32 bit arch
* 2GB on 64 bit arch
*/
#if SIZE_MAX <= 4294967295U
lumiera_config_setdefault ("backend.mmap.window_size = 134217728");
lumiera_config_setdefault ("backend.mmap.window_size = 134217728");
#else
lumiera_config_setdefault ("backend.mmap.window_size = 2147483648");
lumiera_config_setdefault ("backend.mmap.window_size = 2147483648");
#endif
}
long long mmap_window_size = 0;
lumiera_config_number_get ("backend.mmap.window_size", &mmap_window_size);
@ -207,7 +202,6 @@ lumiera_mmap_init (LumieraMMap self, LumieraFile file, off_t start, size_t size)
etruncate:
efile:
lumiera_file_handle_release (file);
lumiera_free (self);
return NULL;
}

9
src/lib/resourcecollector.c → src/backend/resourcecollector.c
View file

@ -32,7 +32,6 @@
llist lumiera_resourcecollector_registry[LUMIERA_RESOURCE_END];
lumiera_mutex lumiera_resourcecollector_lock;
static pthread_once_t lumiera_resourcecollector_once = PTHREAD_ONCE_INIT;
struct lumiera_resourcehandler_struct
{
@ -42,8 +41,8 @@ struct lumiera_resourcehandler_struct
};
static void
lumiera_resourcecollector_init_ (void)
void
lumiera_resourcecollector_init (void)
{
//NOBUG_INIT_FLAG (resourcecollector);
TRACE (resourcecollector_dbg);
@ -74,8 +73,6 @@ lumiera_resourcecollector_run (enum lumiera_resource which, enum lumiera_resourc
{
TRACE (resourcecollector_dbg);
pthread_once (&lumiera_resourcecollector_once, lumiera_resourcecollector_init_);
LUMIERA_MUTEX_SECTION (mutex_sync, &lumiera_resourcecollector_lock)
{
for (enum lumiera_resource_try progress = LUMIERA_RESOURCE_NONE; progress < *iteration; ++*iteration)
@ -119,8 +116,6 @@ lumiera_resourcecollector_run (enum lumiera_resource which, enum lumiera_resourc
LumieraResourcehandler
lumiera_resourcecollector_register_handler (enum lumiera_resource resource, lumiera_resource_handler_fn handler, void* data)
{
pthread_once (&lumiera_resourcecollector_once, lumiera_resourcecollector_init_);
TRACE (resourcecollector_dbg);
LumieraResourcehandler self = lumiera_malloc (sizeof (*self));

10
src/lib/resourcecollector.h → src/backend/resourcecollector.h
View file

@ -93,15 +93,23 @@ enum lumiera_resource_try
* @return indication what the the handler really did (LUMIERA_RESOURCE_NONE when it didn't obey the request)
*/
typedef enum lumiera_resource_try (*lumiera_resource_handler_fn)(enum lumiera_resource_try itr, void* data, void* context);
typedef int (*lumiera_resourcecollector_run_fn) (enum lumiera_resource which, enum lumiera_resource_try* iteration, void* context);
typedef struct lumiera_resourcehandler_struct lumiera_resourcehandler;
typedef lumiera_resourcehandler* LumieraResourcehandler;
/**
* Initialize the Resourcecollector.
* The resourcecollector is singleton and can be used after initialized once.
*/
void
lumiera_resourcecollector_init (void);
/**
* Destroy the resource collector registry.
* Unregisters and deletes all handlers.
* Note that there is no resourcecollector_init() function, initialization is automatic on first use.
*/
void
lumiera_resourcecollector_destroy (void);

2
src/lib/Makefile.am
View file

@ -47,7 +47,6 @@ liblumiera_la_SOURCES = \
$(liblumiera_la_srcdir)/psplay.c \
$(liblumiera_la_srcdir)/mrucache.c \
$(liblumiera_la_srcdir)/time.c \
$(liblumiera_la_srcdir)/resourcecollector.c \
$(liblumiera_la_srcdir)/allocationcluster.cpp \
$(liblumiera_la_srcdir)/lumitime.cpp \
$(liblumiera_la_srcdir)/lifecycle.cpp \
@ -77,7 +76,6 @@ noinst_HEADERS += \
$(liblumiera_la_srcdir)/psplay.h \
$(liblumiera_la_srcdir)/mrucache.h \
$(liblumiera_la_srcdir)/time.h \
$(liblumiera_la_srcdir)/resourcecollector.h \
$(liblumiera_la_srcdir)/ppmpl.h \
$(liblumiera_la_srcdir)/allocationcluster.hpp \
$(liblumiera_la_srcdir)/scoped-holder.hpp \

6
src/lib/error.c
View file

@ -58,10 +58,8 @@ static void
lumiera_error_tls_delete (void* err)
{
if (err)
{
free (((LumieraErrorcontext)err)->extra);
free (err);
}
free (((LumieraErrorcontext)err)->extra);
free(err);
}
static void

33
src/lib/mpool.c
View file

@ -37,10 +37,20 @@
#define MPOOL_C(c) c ## UL
#endif
/*
defaults for the hooks, used when creating mpools
*/
void *(*mpool_malloc_hook)(size_t size) = malloc;
void (*mpool_free_hook)(void *ptr) = free;
/** called after a mpool got initialized */
void (*mpool_init_hook) (MPool self) = NULL;
/** called before a mpool gets destroyed */
void (*mpool_destroy_hook) (MPool self) = NULL;
/*
Cluster and node structures are private
*/
typedef struct mpoolcluster_struct mpoolcluster;
typedef mpoolcluster* MPoolcluster;
typedef const mpoolcluster* const_MPoolcluster;
@ -94,6 +104,12 @@ mpool_init (MPool self, size_t elem_size, unsigned elements_per_cluster, mpool_d
self->elements_free = 0;
self->destroy = dtor;
self->locality = NULL;
self->malloc_hook = mpool_malloc_hook;
self->free_hook = mpool_free_hook;
if (mpool_init_hook)
mpool_init_hook (self);
}
return self;
@ -129,6 +145,9 @@ mpool_destroy (MPool self)
TRACE (mpool_dbg, "%p", self);
if (self)
{
if (mpool_destroy_hook)
mpool_destroy_hook (self);
LLIST_WHILE_TAIL(&self->clusters, cluster)
{
if (self->destroy)
@ -144,7 +163,7 @@ mpool_destroy (MPool self)
llist_unlink_fast_ (cluster);
TRACE (mpool_dbg, "freeing cluster %p" , cluster);
free (cluster);
self->free_hook (cluster);
}
llist_init (&self->freelist);
@ -156,11 +175,19 @@ mpool_destroy (MPool self)
}
MPool
mpool_purge (MPool self)
{
// not UNIMPLEMENTED because valid no-op
PLANNED("To be implemented");
return self;
}
MPool
mpool_cluster_alloc_ (MPool self)
{
MPoolcluster cluster = malloc (self->cluster_size);
MPoolcluster cluster = self->malloc_hook (self->cluster_size);
TRACE (mpool_dbg, "%p", cluster);
if (!cluster)

23
src/lib/mpool.h
View file

@ -77,9 +77,20 @@ struct mpool_struct
unsigned elements_free; /* a counter of free elements is the price we pay to support a reserve() operation */
void* locality;
mpool_destroy_fn destroy;
void *(*malloc_hook)(size_t);
void (*free_hook)(void *);
void* udata; /* free to use by the user, resourcecollector stuff in lumiera*/
};
extern void *(*mpool_malloc_hook)(size_t size);
extern void (*mpool_free_hook)(void *ptr);
/** called after a mpool got initialized */
extern void (*mpool_init_hook) (MPool self);
/** called before a mpool gets destroyed */
extern void (*mpool_destroy_hook) (MPool self);
/*
//index.mpool_init xref:mpool_init[mpool_init()]:: initialize a new memory pool
//mpool [[mpool_init]]
@ -128,6 +139,18 @@ mpool_init (MPool self, size_t elem_size, unsigned elements_per_cluster, mpool_d
MPool
mpool_destroy (MPool self);
/*
//index.mpool_purge xref:mpool_purge[mpool_purge()]:: free unused clusters
//mpool [[mpool_purge]]
//mpool .mpool_purge
//mpool
//mpool TODO
//mpool
//mpool
*/
MPool
mpool_purge (MPool self);
/*
//index.mpool_available xref:mpool_available[mpool_available()]:: query number of free elements

84
src/lib/safeclib.c
View file

@ -21,6 +21,8 @@
#include "lib/error.h"
#include "lib/safeclib.h"
#include "backend/resourcecollector.h"
#include <string.h>
#include <stdlib.h>
#include <pthread.h>
@ -31,12 +33,40 @@
LUMIERA_ERROR_DEFINE (NO_MEMORY, "Out of Memory!");
/* placeholder function until the resourcecollector gets hooked in */
static int
die_no_mem (enum lumiera_resource which, enum lumiera_resource_try* iteration, void* context)
{
(void) which; (void) iteration; (void) context;
LUMIERA_DIE (NO_MEMORY);
return 0; /* not reached */
}
static lumiera_resourcecollector_run_fn lumiera_safeclib_resourcecollector_run_hook = die_no_mem;
void
lumiera_safeclib_set_resourcecollector (void* hook)
{
if (hook)
lumiera_safeclib_resourcecollector_run_hook = (lumiera_resourcecollector_run_fn)hook;
else
lumiera_safeclib_resourcecollector_run_hook = die_no_mem;
}
void*
lumiera_malloc (size_t size)
{
void* o = size ? malloc (size) : NULL;
if (!o)
LUMIERA_DIE (NO_MEMORY);
enum lumiera_resource_try iteration = LUMIERA_RESOURCE_ONE;
void* o = NULL;
if (size)
do
o = malloc (size);
while (!o && lumiera_safeclib_resourcecollector_run_hook (LUMIERA_RESOURCE_MEMORY, &iteration, &size));
return o;
}
@ -44,9 +74,16 @@ lumiera_malloc (size_t size)
void*
lumiera_calloc (size_t n, size_t size)
{
void* o = (n&&size)? calloc (n, size) : NULL;
if (!o)
LUMIERA_DIE (NO_MEMORY);
enum lumiera_resource_try iteration = LUMIERA_RESOURCE_ONE;
void* o = NULL;
size_t gross = n*size;
if (n&&size)
do
o = calloc (n, size);
while (!o && lumiera_safeclib_resourcecollector_run_hook (LUMIERA_RESOURCE_MEMORY, &iteration, &gross));
return o;
}
@ -54,9 +91,13 @@ lumiera_calloc (size_t n, size_t size)
void*
lumiera_realloc (void* ptr, size_t size)
{
void* o = size ? realloc (ptr, size) : NULL;
if (!o)
LUMIERA_DIE (NO_MEMORY);
enum lumiera_resource_try iteration = LUMIERA_RESOURCE_ONE;
void* o = NULL;
if (size)
do
o = realloc (ptr, size);
while (!o && lumiera_safeclib_resourcecollector_run_hook (LUMIERA_RESOURCE_MEMORY, &iteration, &size));
return o;
}
@ -65,14 +106,16 @@ lumiera_realloc (void* ptr, size_t size)
char*
lumiera_strndup (const char* str, size_t len)
{
char* o;
if (str)
o = strndup (str, len);
else
o = strdup ("");
enum lumiera_resource_try iteration = LUMIERA_RESOURCE_ONE;
void* o = NULL;
do
if (str && len)
o = strndup (str, len);
else
o = strdup ("");
while (!o && lumiera_safeclib_resourcecollector_run_hook (LUMIERA_RESOURCE_MEMORY, &iteration, &len));
if (!o)
LUMIERA_DIE (NO_MEMORY);
return o;
}
@ -234,3 +277,12 @@ lumiera_tmpbuf_tr (const char* in, const char* from, const char* to, const char*
return ret;
}
/*
// Local Variables:
// mode: C
// c-file-style: "gnu"
// indent-tabs-mode: nil
// End:
*/

12
src/lib/safeclib.h
View file

@ -29,6 +29,18 @@
*/
LUMIERA_ERROR_DECLARE(NO_MEMORY);
/**
* Install the resourcecollector run hook.
* The resourcecollectr must be hooked into the safeclib at bootup after it got
* initialized and removed from it before shut down. Without resourcecollector
* failed allocations will abort().
* @param hook pointer to the resourcecollector_run function, must be of type
* lumiera_resourcecollector_run_fn but we dont want a dependency on backend in this header
*/
void
lumiera_safeclib_set_resourcecollector (void* hook);
/**
* Allocate memory.
* always succeeds or dies

7
tests/30mmap.tests
View file

@ -10,27 +10,30 @@ TEST "mmap not released should fail" mmap_forget_releasing <<END
return: 134
END
rm ,tmp-filemmap 2>/dev/null
TEST "basic mmap" mmap_simple <<END
return: 0
END
rm ,tmp-filemmap 2>/dev/null
TEST "use mmap twice" mmap_checkout_twice <<END
return: 0
END
rm ,tmp-filemmap 2>/dev/null
TEST "reuse mmap" mmap_checkout_again <<END
return: 0
END
TEST "grow file" mmap_grow_existing_file <<END
return: 0
END
TEST "readonly file" mmap_readonly_file <<END
return: 0
END
rm ,tmp-filemmap 2>/dev/null

0
tests/20resourcecollector.tests → tests/30resourcecollector.tests
View file

5
tests/Makefile.am
View file

@ -73,9 +73,10 @@ test_filemmap_LDADD = $(LUMIERA_LIBS) liblumierabackend.la liblumieracommon.la l
# FIXME stray dependencies on proc
check_PROGRAMS += test-resourcecollector
test_resourcecollector_SOURCES = $(tests_srcdir)/library/test-resourcecollector.c
test_resourcecollector_SOURCES = $(tests_srcdir)/backend/test-resourcecollector.c
test_resourcecollector_CPPFLAGS = $(AM_CPPFLAGS) -std=gnu99 -Wall -Werror -I$(top_srcdir)/src/
test_resourcecollector_LDADD = $(LUMIERA_LIBS) liblumiera.la
test_resourcecollector_LDADD = $(LUMIERA_LIBS) liblumiera.la liblumierabackend.la liblumieraproc.la liblumieracommon.la
# FIXME stray dependencies on proc
check_PROGRAMS += test-slist
test_slist_SOURCES = $(tests_srcdir)/library/test-slist.c

4
tests/library/test-resourcecollector.c → tests/backend/test-resourcecollector.c
View file

@ -19,7 +19,7 @@
Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
*/
#include "lib/resourcecollector.h"
#include "backend/resourcecollector.h"
#include "tests/test.h"
@ -50,6 +50,7 @@ TESTS_BEGIN
TEST ("basic")
{
lumiera_resourcecollector_init ();
lumiera_resourcecollector_register_handler (LUMIERA_RESOURCE_MEMORY, test_memory_handler, NULL);
lumiera_resourcecollector_destroy ();
}
@ -57,6 +58,7 @@ TEST ("basic")
TEST ("memory_collection_mockup")
{
REQUIRE (argv[2]);
lumiera_resourcecollector_init ();
lumiera_resourcecollector_register_handler (LUMIERA_RESOURCE_MEMORY, test_memory_handler, NULL);