ichthyo/LUMIERA.clone
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

WIP: mmap management first go

Browse source 
Quite some code which was hold back in favor of the config and plugin stuff
implements:

 * mmapcache: mru cache for unused memory mappings
 * mmap: single mmaped areas
 * mmapings: manages mmaps established for one filedescriptor
This commit is contained in:
Christian Thaeter 2008-11-07 15:36:18 +01:00
parent 8c2b98a2e1
commit e938b36071
17 changed files with 801 additions and 45 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

5
doc/devel/nobug_flags.txt
View file

@ -20,7 +20,10 @@ all # global logging
filedescriptor # internal filedescriptors
filehandle # posix filehandles
filehandlecache # mrucache for filehandles
map_all # file mapping subsystem
mmap_all # file mapping subsystem
mmap # mmap objects
mmapings # mmap range containers
mmapcache # mmap range containers
cache_all # caching subsystem
scheduler_all # all scheduler
threads # threadpool management

12
src/backend/Makefile.am
View file

@ -37,8 +37,10 @@ liblumibackend_a_SOURCES = \
$(liblumibackend_a_srcdir)/config_wordlist.c \
$(liblumibackend_a_srcdir)/configentry.c \
$(liblumibackend_a_srcdir)/configitem.c \
$(liblumibackend_a_srcdir)/config_lookup.c
$(liblumibackend_a_srcdir)/config_lookup.c \
$(liblumibackend_a_srcdir)/mmap.c \
$(liblumibackend_a_srcdir)/mmapings.c \
$(liblumibackend_a_srcdir)/mmapcache.c
noinst_HEADERS += \
$(liblumibackend_a_srcdir)/mediaaccessfacade.cpp \
@ -54,5 +56,7 @@ noinst_HEADERS += \
$(liblumibackend_a_srcdir)/config.h \
$(liblumibackend_a_srcdir)/configentry.h \
$(liblumibackend_a_srcdir)/configitem.h \
$(liblumibackend_a_srcdir)/config_lookup.h
$(liblumibackend_a_srcdir)/config_lookup.h \
$(liblumibackend_a_srcdir)/mmap.h \
$(liblumibackend_a_srcdir)/mmapings.h \
$(liblumibackend_a_srcdir)/mmapcache.h

47
src/backend/backend.c
View file

@ -22,25 +22,65 @@
#include "backend/backend.h"
#include "backend/filehandlecache.h"
#include "backend/filedescriptor.h"
#include "backend/mmapcache.h"
#include <unistd.h>
#include <sys/resource.h>
//NOBUG_DEFINE_FLAG_PARENT (backend, lumiera); TODO
NOBUG_DEFINE_FLAG (backend);
NOBUG_DEFINE_FLAG_PARENT (file_all, backend);
NOBUG_DECLARE_FLAG (file);
NOBUG_DECLARE_FLAG (mmap_all);
NOBUG_DECLARE_FLAG (mmap);
NOBUG_DECLARE_FLAG (mmapings);
size_t lumiera_backend_pagesize;
int
lumiera_backend_init (void)
{
NOBUG_INIT_FLAG (backend);
NOBUG_INIT_FLAG (file_all);
NOBUG_INIT_FLAG (file);
NOBUG_INIT_FLAG (mmap_all);
NOBUG_INIT_FLAG (mmap);
NOBUG_INIT_FLAG (mmapings);
TRACE (backend);
lumiera_filedescriptor_registry_init ();
int max_entries = 10; TODO("determine by sysconf (_SC_OPEN_MAX) minus some (big) safety margin "
"add some override to run tests with few filehandles");
lumiera_backend_pagesize = sysconf(_SC_PAGESIZE);
TODO ("add config options to override following defaults");
/* roughly 2/3 of all availables filehandles are managed by the backend */
int max_entries = (sysconf (_SC_OPEN_MAX)-10)*2/3;
INFO (backend, "using %d filehandles", max_entries);
lumiera_filehandlecache_new (max_entries);
struct rlimit as_limit;
getrlimit (RLIMIT_AS, &as_limit);
if (as_limit.rlim_cur == RLIM_INFINITY)
{
#if SIZE_MAX <= 4294967295UL
INFO (backend, "address space not limited, backend will mmap at most 1.5GiB");
as_limit.rlim_cur = 1610612736U;
#else
INFO (backend, "address space not limited, backend will mmap at most 192TiB");
as_limit.rlim_cur = 211106232532992U;
#endif
}
else
{
INFO (backend, "address space limited to %luMiB", as_limit.rlim_cur/1024/1024);
}
lumiera_mmapcache_new (as_limit.rlim_cur);
return 0;
}
@ -48,6 +88,7 @@ void
lumiera_backend_destroy (void)
{
TRACE (backend);
lumiera_filedescriptor_registry_destroy ();
lumiera_mmapcache_delete ();
lumiera_filehandlecache_delete ();
lumiera_filedescriptor_registry_destroy ();
}

5
src/backend/backend.h
View file

@ -24,8 +24,9 @@
#include <nobug.h>
NOBUG_DECLARE_FLAG (backend);
NOBUG_DECLARE_FLAG (file_all);
NOBUG_DECLARE_FLAG (file);
extern size_t lumiera_backend_pagesize;
int
lumiera_backend_init (void);

19
src/backend/file.c
View file

@ -31,15 +31,19 @@
NOBUG_DEFINE_FLAG_PARENT (file, file_all);
LUMIERA_ERROR_DEFINE(FILE_CHANGED, "File changed unexpected");
LUMIERA_ERROR_DEFINE (FILE_CHANGED, "File changed unexpected");
LumieraFile
lumiera_file_init (LumieraFile self, const char* name, int flags)
lumiera_file_init (LumieraFile self, const char* name, int flags, size_t chunksize)
{
TRACE (file);
if (!(self->descriptor = lumiera_filedescriptor_acquire (name, flags)))
return NULL;
if (chunksize && !self->descriptor->mmapings)
self->descriptor->mmapings = lumiera_mmapings_new (self, chunksize);
self->name = lumiera_strndup (name, PATH_MAX);
return self;
@ -56,11 +60,11 @@ lumiera_file_destroy (LumieraFile self)
LumieraFile
lumiera_file_new (const char* name, int flags)
lumiera_file_new (const char* name, int flags, size_t chunksize)
{
TRACE (file);
LumieraFile self = lumiera_malloc (sizeof (lumiera_file));
return lumiera_file_init (self, name, flags);
return lumiera_file_init (self, name, flags, chunksize);
}
void
@ -128,3 +132,10 @@ lumiera_file_handle_release (LumieraFile self)
lumiera_filehandlecache_checkin (lumiera_fhcache, self->descriptor->handle);
}
}
LumieraMMapings
lumiera_file_mmapings (LumieraFile self)
{
REQUIRE (self->descriptor->mmapings, "mmapings not initialized")
return self->descriptor->mmapings;
}

22
src/backend/file.h
View file

@ -48,7 +48,8 @@ typedef lumiera_file* LumieraFile;
#include "backend/filehandle.h"
#include "backend/filedescriptor.h"
#include "backend/file.h"
#include "backend/mmapings.h"
#define LUMIERA_FILE_READONLY (O_RDONLY | O_LARGEFILE | O_NOATIME)
#define LUMIERA_FILE_READWRITE (O_RDWR | O_LARGEFILE | O_NOATIME)
@ -72,12 +73,14 @@ struct lumiera_file_struct
* @return self
*/
LumieraFile
lumiera_file_init (LumieraFile self, const char* name, int flags);
lumiera_file_init (LumieraFile self, const char* name, int flags, size_t chunksize);
/**
* Destroy a file structure.
* frees all associated resources, releases the filedescriptor etc.
* @param self file structure to be destroyed
* @param chunksize allocation/mmaping granularity, must be 2's exponent of pagesize
* only used at the first access to a file and ignored for subsequnet accesses
* @return self
*/
LumieraFile
@ -87,10 +90,12 @@ lumiera_file_destroy (LumieraFile self);
* Allocate a new file structure.
* @param name filename
* @param flags open flags
* @param chunksize allocation/mmaping granularity, must be 2's exponent of pagesize
* only used at the first access to a file and ignored for subsequnet accesses
* @return new file structure
*/
LumieraFile
lumiera_file_new (const char* name, int flags);
lumiera_file_new (const char* name, int flags, size_t chunksize);
/**
* Frees a file structure.
@ -119,5 +124,16 @@ lumiera_file_handle_acquire (LumieraFile self);
void
lumiera_file_handle_release (LumieraFile self);
static inline LumieraFiledescriptor
lumiera_file_descriptor (LumieraFile self)
{
return self->descriptor;
}
LumieraMMapings
lumiera_file_mmapings (LumieraFile self);
#endif

35
src/backend/filedescriptor.h
View file

@ -36,6 +36,7 @@ typedef lumiera_filedescriptor* LumieraFiledescriptor;
#include "backend/filehandle.h"
#include "backend/file.h"
#include "backend/mmapings.h"
/**
* @file
@ -43,19 +44,35 @@ typedef lumiera_filedescriptor* LumieraFiledescriptor;
* Filedescriptors are the underlying working horse in accessing files.
* All information associated with managing a file is kept here.
*/
struct lumiera_filedescriptor_struct
{
psplaynode node; /* node for the lookup tree */
struct stat stat; /* create after first open, maintained metadata */
int flags; /* open flags, must be masked for reopen */
lumiera_mutex lock; /* locks operations on this file descriptor */
unsigned refcount; /* reference counter, all users sans registry */
/** node for the lookup tree */
psplaynode node;
/** create after first open, maintained metadata */
struct stat stat;
/**
* files which are written are rounded up to (the next chunk boundary)
* by the mmaping backend and will be ftruncated to the realsize on close.
*/
off_t realsize;
/** open flags, must be masked for reopen */
int flags;
/** locks operations on this file descriptor */
lumiera_mutex lock;
/** reference counter, all users (except registry) */
unsigned refcount;
/** Associated posix filehandle */
LumieraFilehandle handle;
//LumieraFileMap mappings;
/** established memory mappings */
LumieraMMapings mmapings;
//LumieraWriteBuffer writebuffer;
};

8
src/backend/filehandlecache.h
View file

@ -78,14 +78,6 @@ lumiera_filehandlecache_delete (void);
LumieraFilehandle
lumiera_filehandlecache_handle_acquire (LumieraFilehandlecache self, LumieraFiledescriptor desc);
/**
* Add filehande back to cache, the filehandle becomes subject of aging.
* @param self pointer to the cache
* @param handle filehandle to be put back
*/
void
lumiera_filehandlecache_add_filehandle (LumieraFilehandlecache self, LumieraFilehandle handle);
/**
* Remove a filehandle from cache aging
* Filehandles which are subject of cache aging must be checked out before they can be used.

260
src/backend/mmap.c Normal file
View file

@ -0,0 +1,260 @@
/*
mmap.c - memory mapped acces to files
Copyright (C) Lumiera.org
2008, Christian Thaeter <ct@pipapo.org>
This program is free software; you can redistribute it and/or
modify it under the terms of the GNU General Public License as
published by the Free Software Foundation; either version 2 of the
License, or (at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program; if not, write to the Free Software
Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
*/
#include "lib/safeclib.h"
#include "backend/mmap.h"
#include "backend/mmapcache.h"
#include <unistd.h>
#include <sys/mman.h>
/**
* @file
*
*/
NOBUG_DEFINE_FLAG_PARENT (mmap_all, backend);
NOBUG_DEFINE_FLAG_PARENT (mmap, mmap_all);
LUMIERA_ERROR_DEFINE (MMAP_NWRITE, "Backing file not writeable");
LUMIERA_ERROR_DEFINE (MMAP_SPACE, "Address space exhausted");
/**
* global mmap registry/cache
*
*/
/**
* default size for the mmaping window
* 128MB on 32 bit arch
* 2GB on 64 bit arch
*/
#if SIZE_MAX <= 4294967295U
static size_t mmap_window_size = 134217728UL;
#else
static size_t mmap_window_size = 2147483648UL;
#endif
LumieraMMap
lumiera_mmap_init (LumieraMMap self, LumieraFile file, LList acquirer, off_t start, size_t size, size_t chunksize)
{
TRACE (mmap);
REQUIRE (self);
REQUIRE (file);
REQUIRE (acquirer);
REQUIRE (llist_is_empty (acquirer));
REQUIRE (start >= 0);
REQUIRE (size);
LumieraFiledescriptor descriptor = file->descriptor;
int fd = lumiera_file_handle_acquire (file);
TRACE (mmap, "got fd %d", fd);
if (fd == -1)
goto efile;
void* addr = NULL;
off_t begin = 0;
size_t length = 0;
/**
* Maintaining the right[tm] mmaping size is a bit tricky:
* - We have the default mmap_window_size which will be backed off when address space gets exhausted
* - When a biggier size is requested we have to fullfill it
* - The last mmaped chunk of a file can be as small as possible when the file is readonly
* - When the file is writeable, the last chunk should be rounded up to chunksize
* - All boundaries will be aligned up/down to chunk boundaries
* - Requests beyond the file end must ftruncate and map additional pages
* - Create the 'refmap' which contains a refounter per chunk
**/
while (!addr)
{
/**
* Recovering address space strategies:
* mmap() will fail when too much memory got mmaped after some time which is then
* recovered in the following way
* 1. create a new mmap while the cachelimit is not reached.
* 2. All unused mmaps are kept in a mrucache, drop the oldest one.
* mmap() still fails..
* 3.a When the intented mmaping size is the same as mmap_window_size then reduce (/2) the window size and retry.
* 3.b When the intented mmaping size was biggier than mmap_window_size then free more mmaps from the cache.
* 4 When the cache is empty (that means all mmaps in use), scan the mmaps in use if they can be reduced
* mmap_window_size is already reduced now (half of refmap free from either end)
**/
enum {
FIRST_TRY,
DROP_FROM_CACHE,
REDUCE_WINDOW,
REDUCE_IN_USE,
GIVE_UP
} strategy = FIRST_TRY;
switch (strategy++)
{
case FIRST_TRY:
/* align begin and end to chunk boundaries */
begin = start & ~(chunksize-1);
length = ((start+size+chunksize-1) & ~(chunksize-1)) - begin;
if (begin+(off_t)length > descriptor->stat.st_size)
{
/* request past the end */
if ((descriptor->flags & O_ACCMODE) == O_RDWR)
{
/* extend file (writeable) */
if (ftruncate (fd, begin+length) == -1)
{
LUMIERA_ERROR_SET (mmap, ERRNO);
goto etruncate;
};
descriptor->stat.st_size = begin+length;
descriptor->realsize = start+size;
}
else
{
LUMIERA_ERROR_SET (mmap, MMAP_NWRITE);
goto ewrite;
}
}
else if (length < mmap_window_size)
length = mmap_window_size;
if ((descriptor->flags & O_ACCMODE) == O_RDONLY)
/* The last mmaped chunk of a file can be as small as possible when the file is readonly */
length = start+size - begin;
break;
case DROP_FROM_CACHE:
TRACE (mmap, "drop a mapping from cache");
UNIMPLEMENTED ("mmap cache drop");
break;
case REDUCE_WINDOW:
NOTICE (mmap, "mmaping window reduced to NN MB");
UNIMPLEMENTED ("mmap window reduce");
break;
case REDUCE_IN_USE:
NOTICE (mmap, "reduce mmapings in use");
UNIMPLEMENTED ("mmapings in use reduce");
break;
case GIVE_UP:
LUMIERA_ERROR_SET (mmap, MMAP_SPACE);
goto espace;
}
addr = mmap (NULL,
length,
(descriptor->flags & O_ACCMODE) == O_RDONLY ? PROT_READ : PROT_READ|PROT_WRITE,
MAP_SHARED,
fd,
begin);
}
llist_init (&self->cachenode);
llist_init (&self->searchnode);
self->start = begin;
self->size = length;
self->address = addr;
self->refmap = lumiera_calloc (length/chunksize, sizeof (unsigned short));
llist_insert_head (&self->cachenode, acquirer);
lumiera_mmapcache_announce (lumiera_mcache, self);
lumiera_file_handle_release (file);
return self;
espace:
etruncate:
ewrite:
efile:
lumiera_file_handle_release (file);
free (self);
return NULL;
}
LumieraMMap
lumiera_mmap_new (LumieraFile file, LList acquirer, off_t start, size_t size, size_t chunksize)
{
TRACE (mmap);
LumieraMMap self = lumiera_mmapcache_mmap_acquire (lumiera_mcache);
if (lumiera_mmap_init (self, file, acquirer, start, size, chunksize))
return self;
else
{
free (self);
return NULL;
}
}
void
lumiera_mmap_delete (LumieraMMap self)
{
TRACE (mmap);
if (self)
{
//LUMIERA_MUTEX_SECTION (mmapings, self->rh, &self->lock)
lumiera_mmapcache_forget (lumiera_mcache, self);
llist_unlink (&self->searchnode); TODO ("must lock mmapings -> deadlock");
munmap (self->address, self->size);
free (self->refmap);
free (self);
}
}
void*
lumiera_mmap_destroy_node (LList node)
{
TRACE (mmap);
REQUIRE (llist_is_empty (node));
LumieraMMap self = (LumieraMMap)node;
lumiera_mmapcache_forget (lumiera_mcache, self);
llist_unlink (&self->searchnode); TODO ("must lock mmapings -> deadlock");
munmap (self->address, self->size);
free (self->refmap);
return self;
}
/*
// Local Variables:
// mode: C
// c-file-style: "gnu"
// indent-tabs-mode: nil
// End:
*/

6
src/backend/mmap.h
View file

@ -63,8 +63,6 @@ struct lumiera_mmap_struct
/** array with refcounters per page **/
unsigned short* refmap;
/** 0 when this mmap is in cache, else the count of attached owners **/
unsigned use_cnt;
//RESOURCE_HANDLE (rh);
};
@ -75,9 +73,13 @@ lumiera_mmap_init (LumieraMMap self, LumieraFile file, LList acquirer, off_t sta
LumieraMMap
lumiera_mmap_new (LumieraFile file, LList acquirer, off_t start, size_t size, size_t chunksize);
void
lumiera_mmap_delete (LumieraMMap self);
void*
lumiera_mmap_destroy_node (LList node);
#endif
/*
// Local Variables:

164
src/backend/mmapcache.c Normal file
View file

@ -0,0 +1,164 @@
/*
mmapcache.c - handle aging of mmap objects
Copyright (C) Lumiera.org
2008, Christian Thaeter <ct@pipapo.org>
This program is free software; you can redistribute it and/or
modify it under the terms of the GNU General Public License as
published by the Free Software Foundation; either version 2 of the
License, or (at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program; if not, write to the Free Software
Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
*/
#include "lib/safeclib.h"
#include "backend/mmapcache.h"
/**
* @file
*
*/
NOBUG_DEFINE_FLAG_PARENT (mmapcache, mmap_all);
LumieraMMapcache lumiera_mcache = NULL;
void
lumiera_mmapcache_new (size_t limit)
{
TRACE (mmapcache);
lumiera_mcache = lumiera_malloc (sizeof (*lumiera_mcache));
lumiera_mrucache_init (&lumiera_mcache->cache, lumiera_mmap_destroy_node);
lumiera_mcache->limit = limit;
lumiera_mcache->total = 0;
lumiera_mcache->cached = 0;
lumiera_mutex_init (&lumiera_mcache->lock, "mmapcache", &NOBUG_FLAG (mmapcache));
}
void
lumiera_mmapcache_delete (void)
{
if (lumiera_mcache)
{
REQUIRE (lumiera_mcache->total == lumiera_mcache->cached, "MMaps still checked out at shutdown");
lumiera_mrucache_destroy (&lumiera_mcache->cache);
lumiera_mutex_destroy (&lumiera_mcache->lock, &NOBUG_FLAG (mmapcache));
free (lumiera_mcache);
lumiera_mcache = NULL;
}
}
void*
lumiera_mmapcache_mmap_acquire (LumieraMMapcache self)
{
void* map = NULL;
LUMIERA_MUTEX_SECTION (mmapcache, &self->lock)
{
map = lumiera_mrucache_pop (&self->cache);
}
if (!map)
{
map = lumiera_malloc (sizeof (*self));
TRACE (mmapcache, "allocated new mmap");
}
else
{
TRACE (mmapcache, "poped mmap from cache");
}
return map;
}
void
lumiera_mmapcache_announce (LumieraMMapcache self, LumieraMMap map)
{
LUMIERA_MUTEX_SECTION (mmapcache, &self->lock)
{
self->total += map->size;
}
}
void
lumiera_mmapcache_forget (LumieraMMapcache self, LumieraMMap map)
{
LUMIERA_MUTEX_SECTION (mmapcache, &self->lock)
{
if (!llist_is_empty (&map->cachenode))
{
TODO ("cached stats");
llist_unlink (&map->cachenode);
}
self->total -= map->size;
}
}
#if 0
int
lumiera_mmapcache_age (LumieraMMapcache self)
{
TRACE (mmapcache);
int ret = 0;
LUMIERA_MUTEX_SECTION (mmapcache, &self->lock)
{
ret = lumiera_mrucache_age (&self->cache, 10); TODO ("age nelem == 20%(configureable) of the cache");
}
return ret;
}
#endif
LumieraMMap
lumiera_mmapcache_checkout (LumieraMMapcache self, LumieraMMap handle)
{
TRACE (mmapcache);
LUMIERA_MUTEX_SECTION (mmapcache, &self->lock)
{
TODO ("cached stats");
lumiera_mrucache_checkout (&self->cache, &handle->cachenode);
}
return handle;
}
void
lumiera_mmapcache_checkin (LumieraMMapcache self, LumieraMMap handle)
{
TRACE (mmapcache);
LUMIERA_MUTEX_SECTION (mmapcache, &self->lock)
{
TODO ("cached stats");
lumiera_mrucache_checkin (&self->cache, &handle->cachenode);
}
}
/*
// Local Variables:
// mode: C
// c-file-style: "gnu"
// indent-tabs-mode: nil
// End:
*/

7
src/backend/mmapcache.h
View file

@ -73,9 +73,9 @@ lumiera_mmapcache_delete (void);
* when mmaped_limit is reached, the oldest mmap object gets dropped else a new allocated object
* is returned
* @param self pointer to the cache
* @return the new mmap
* @return the new uninitialized mmap
*/
LumieraMMap
void*
lumiera_mmapcache_mmap_acquire (LumieraMMapcache self);
@ -91,8 +91,7 @@ lumiera_mmapcache_announce (LumieraMMapcache self, LumieraMMap map);
/**
* Remove a mmap object from the cache quotas.
* Update the statistics kept in the cache,
* called by mmap's destructor only
* Update the statistics kept in the cache, remove it from the cache.
* @param self pointer to the cache
* @param map object to be removed
*/

158
src/backend/mmapings.c Normal file
View file

@ -0,0 +1,158 @@
/*
mmapings.c - manage ranges of mmaped areas on a filedescriptor
Copyright (C) Lumiera.org
2008, Christian Thaeter <ct@pipapo.org>
This program is free software; you can redistribute it and/or
modify it under the terms of the GNU General Public License as
published by the Free Software Foundation; either version 2 of the
License, or (at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program; if not, write to the Free Software
Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
*/
#include "lib/mutex.h"
#include "lib/safeclib.h"
#include "backend/mmapings.h"
#include "backend/mmapcache.h"
/**
* @file
*
*/
NOBUG_DEFINE_FLAG_PARENT (mmapings, mmap_all);
LumieraMMapings
lumiera_mmapings_init (LumieraMMapings self, LumieraFile file, size_t chunksize)
{
TRACE (mmapings);
REQUIRE (!file->descriptor->mmapings);
llist_init (&self->mmaps);
self->descriptor = file->descriptor;
self->chunksize = chunksize;
lumiera_mutex_init (&self->lock, "mmapings", &NOBUG_FLAG(mmapings));
return self;
}
LumieraMMapings
lumiera_mmapings_destroy (LumieraMMapings self)
{
TRACE (mmapings);
if (!self)
return NULL;
LLIST_WHILE_TAIL (&self->mmaps, node)
{
LumieraMMap mmap = LLIST_TO_STRUCTP (node, lumiera_mmap, searchnode);
lumiera_mmap_delete (mmap);
}
lumiera_mutex_destroy (&self->lock, &NOBUG_FLAG(mmapings));
return self;
}
LumieraMMapings
lumiera_mmapings_new (LumieraFile file, size_t chunksize)
{
LumieraMMapings self = lumiera_malloc (sizeof (*self));
return lumiera_mmapings_init (self, file, chunksize);
}
void
lumiera_mmapings_delete (LumieraMMapings self)
{
TRACE (mmapings);
free (lumiera_mmapings_destroy (self));
}
LumieraMMap
lumiera_mmapings_mmap_acquire (LumieraMMapings self, LumieraFile file, LList acquirer, off_t start, size_t size)
{
TRACE (mmapings);
LumieraMMap ret = NULL;
LUMIERA_MUTEX_SECTION (mmapings, &self->lock)
{
REQUIRE (llist_is_empty (acquirer));
/* find best matching mmap, crude way */
LLIST_FOREACH (&self->mmaps, node)
{
LumieraMMap mmap = LLIST_TO_STRUCTP (node, lumiera_mmap, searchnode);
if (mmap->size >= size && mmap->start <= start && mmap->start+mmap->size >= start+size)
{
ret = mmap;
break;
}
}
if (!ret)
{
/* create new mmap */
TRACE (mmapings, "mmap not found, creating", mmap);
ret = lumiera_mmap_new (file, acquirer, start, size, self->chunksize);
llist_insert_head (&self->mmaps, &ret->searchnode);
TODO ("sort search list");
}
else
{
/* found mmap */
UNIMPLEMENTED ("reuse existing mmap");
//lumiera_mmapcache_checkout (lumiera_mcache, self);
// checkout add acquirer
}
}
return ret;
}
void
lumiera_mmapings_release_mmap (LumieraMMapings self, LList acquirer, LumieraMMap map)
{
TRACE (mmapings);
LUMIERA_MUTEX_SECTION (mmapings, &self->lock)
{
llist_unlink (acquirer);
if (llist_is_empty (&map->cachenode))
{
TRACE (mmapings, "checkin");
lumiera_mmapcache_checkin (lumiera_mcache, map);
}
}
}
/*
// Local Variables:
// mode: C
// c-file-style: "gnu"
// indent-tabs-mode: nil
// End:
*/

3
src/backend/mmapings.h
View file

@ -45,9 +45,6 @@ struct lumiera_mmapings_struct
/** mmaped ranges are kept in an list sorted by the size of the mmaping, might be improved to a tree someday **/
llist mmaps;
/** sum of all mmaped areas, areas might be overlapping **/
// size_t vsz;
/**
* chunkssize is the smallest granularity which is used for mmapping files, it
* should reflect the intended file usage, that is 'pagesize' for small or non growing

5
tests/Makefile.am
View file

@ -74,4 +74,9 @@ test_interfaces_CPPFLAGS = $(AM_CPPFLAGS) -std=gnu99 -Wall -Werror
test_interfaces_LDADD = liblumibackend.a liblumiera.a $(LUMIERA_PLUGIN_LIBS) $(NOBUGMT_LUMIERA_LIBS)
test_interfaces_DEPENDENCIES = examplepluginc.la liblumibackend.a liblumiera.a
check_PROGRAMS += test-filemmap
test_filemmap_SOURCES = $(tests_srcdir)/backend/test-filemmap.c
test_filemmap_CPPFLAGS = $(AM_CPPFLAGS) -std=gnu99 -Wall -Werror -I$(top_srcdir)/src/
test_filemmap_LDADD = liblumibackend.a liblumiera.a -lnobugmt -lpthread -ldl -lm
TESTS = $(tests_srcdir)/test.sh

4
tests/backend/test-filehandles.c
View file

@ -31,7 +31,7 @@ TESTS_BEGIN
TEST ("basic")
{
lumiera_backend_init ();
LumieraFile file = lumiera_file_new (",tmp_testfile", LUMIERA_FILE_CREATE);
LumieraFile file = lumiera_file_new (",tmp_testfile", LUMIERA_FILE_CREATE, 4096);
/* get the filehandle */
int fd = lumiera_file_handle_acquire (file);
@ -58,7 +58,7 @@ TEST ("more")
/*create 100 files*/
for (int i=0; i<100; ++i)
{
files[i]= lumiera_file_new (lumiera_tmpbuf_snprintf (256, ",tmpdir/testfile%d", i), LUMIERA_FILE_CREATE);
files[i]= lumiera_file_new (lumiera_tmpbuf_snprintf (256, ",tmpdir/testfile%d", i), LUMIERA_FILE_CREATE, 4096);
}
/* get the filehandles, this gross overallocates filehandles */

86
tests/backend/test-filemmap.c Normal file
View file

@ -0,0 +1,86 @@
/*
test-files.c - test file management
Copyright (C) Lumiera.org
2008, Christian Thaeter <ct@pipapo.org>
This program is free software; you can redistribute it and/or
modify it under the terms of the GNU General Public License as
published by the Free Software Foundation; either version 2 of the
License, or (at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program; if not, write to the Free Software
Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
*/
//#include <stdio.h>
//#include <string.h>
#include "lib/llist.h"
#include "backend/backend.h"
#include "backend/file.h"
#include "backend/filedescriptor.h"
#include "backend/mmapings.h"
#include "backend/mmap.h"
#include "tests/test.h"
TESTS_BEGIN
TEST ("detail_usage")
{
lumiera_backend_init ();
LumieraFile file = lumiera_file_new (",tmp-filemmap", LUMIERA_FILE_CREATE, 4096);
LumieraMMapings mmaps = lumiera_file_mmapings (file);
llist user;
llist_init (&user);
LumieraMMap mmap = lumiera_mmapings_mmap_acquire (mmaps, file, &user, 0, 100);
lumiera_mmapings_release_mmap (mmaps, &user, mmap);
lumiera_file_delete (file);
lumiera_backend_destroy ();
}
#if 0
TEST ("refactored_usage")
{
LumieraFile file = lumiera_file_new ("filename", mode);
LumieraFrameIndex index = lumiera_frameindex_new ("indexfilename", file /*, indexing engine*/);
LumieraFrame frame = lumiera_frameindex_frame (index, 123);
//TAG+HINTS could be NEXT|PREV|EXACT|NEAREST|HARD|SOFT
lumiera_frame_prefetch (index, 123);
lumiera_frame_release (frame);
lumiera_frameindex_delete (index);
lumiera_file_free (file);
}
#endif
TESTS_END