Filehandling

* add 'backend' for backend global initialization/destruction
  and common functions.
* WIP: 'file' acts as interface to named files, provides posix
  filehandles with and acquire/release function pair.
* 'filehandle' manages posix filehandles internally, refcounted.
* WIP: 'filehandlecache' keeps unused filehandles in a MRU cache

* WIP: test for filehandle management
* improved filedescriptor test
This commit is contained in:
Christian Thaeter 2008-04-07 06:47:57 +02:00
parent b11ff40fbe
commit 232afe77e9
13 changed files with 415 additions and 60 deletions

53
src/backend/backend.c Normal file
View file

@ -0,0 +1,53 @@
/*
backend - common lumiera backend things
Copyright (C) Lumiera.org
2008, Christian Thaeter <ct@pipapo.org>
This program is free software; you can redistribute it and/or
modify it under the terms of the GNU General Public License as
published by the Free Software Foundation; either version 2 of the
License, or (at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program; if not, write to the Free Software
Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
*/
#include "backend/backend.h"
#include "backend/filehandlecache.h"
#include "backend/filedescriptor.h"
//NOBUG_DEFINE_FLAG_PARENT (backend, lumiera); TODO
NOBUG_DEFINE_FLAG (backend);
NOBUG_DEFINE_FLAG_PARENT (file_all, backend);
int
lumiera_backend_init (void)
{
NOBUG_INIT_FLAG (backend);
NOBUG_INIT_FLAG (file_all);
NOBUG_INIT_FLAG (file);
TRACE (backend);
lumiera_filedescriptor_registry_init ();
int max_entries = 10; TODO("determine by sysconf (_SC_OPEN_MAX) minus some (big) safety margin "
"add some override to run tests with few filehandles");
lumiera_filehandlecache_new (max_entries);
return 0;
}
void
lumiera_backend_destroy (void)
{
TRACE (backend);
lumiera_filedescriptor_registry_destroy ();
lumiera_filehandlecache_delete ();
}

36
src/backend/backend.h Normal file
View file

@ -0,0 +1,36 @@
/*
backend - common lumiera backend things
Copyright (C) Lumiera.org
2008, Christian Thaeter <ct@pipapo.org>
This program is free software; you can redistribute it and/or
modify it under the terms of the GNU General Public License as
published by the Free Software Foundation; either version 2 of the
License, or (at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program; if not, write to the Free Software
Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
*/
#ifndef LUMIERA_BACKEND_H
#define LUMIERA_BACKEND_H
#include <nobug.h>
NOBUG_DECLARE_FLAG (backend);
NOBUG_DECLARE_FLAG (file_all);
NOBUG_DECLARE_FLAG (file);
int
lumiera_backend_init (void);
void
lumiera_backend_destroy (void);
#endif

View file

@ -23,17 +23,18 @@
#include "lib/safeclib.h"
#include "backend/file.h"
#include "backend/filehandlecache.h"
#include "backend/filedescriptor.h"
#include <limits.h>
NOBUG_DEFINE_FLAG (lumiera_file);
NOBUG_DEFINE_FLAG_PARENT (file, file_all);
LumieraFile
lumiera_file_init (LumieraFile self, const char* name, int flags)
{
if (!lumiera_filedescriptor_acquire (&self->descriptor, name, flags))
TRACE (file);
if (!(self->descriptor = lumiera_filedescriptor_acquire (name, flags)))
return NULL;
self->name = lumiera_strndup (name, PATH_MAX);
@ -43,7 +44,8 @@ lumiera_file_init (LumieraFile self, const char* name, int flags)
LumieraFile
lumiera_file_destroy (LumieraFile self)
{
lumiera_reference_destroy (&self->descriptor);
TRACE (file);
lumiera_filedescriptor_release (self->descriptor);
free ((void*)self->name);
return self;
}
@ -52,6 +54,7 @@ lumiera_file_destroy (LumieraFile self)
LumieraFile
lumiera_file_new (const char* name, int flags)
{
TRACE (file);
LumieraFile self = lumiera_malloc (sizeof (lumiera_file));
return lumiera_file_init (self, name, flags);
}
@ -59,6 +62,48 @@ lumiera_file_new (const char* name, int flags)
void
lumiera_file_delete (LumieraFile self)
{
TRACE (file);
free (lumiera_file_destroy (self));
}
int
lumiera_file_handle_acquire (LumieraFile self)
{
TRACE (file);
REQUIRE (self);
REQUIRE (self->descriptor);
REQUIRE (fhcache);
LUMIERA_MUTEX_SECTION (file, self->descriptor->rh, &self->descriptor->lock)
{
if (!self->descriptor->handle)
/* no handle yet, get a new one */
lumiera_filehandlecache_handle_acquire (fhcache, self->descriptor);
else
lumiera_filehandlecache_checkout (fhcache, self->descriptor->handle);
if (self->descriptor->handle->fd == -1)
{
TODO ("stat handling/update");
self->descriptor->handle->fd = open (self->name, self->descriptor->flags);
if (self->descriptor->handle->fd == -1)
{
LUMIERA_ERROR_SET (file, ERRNO);
}
}
}
return self->descriptor->handle->fd;
}
void
lumiera_file_handle_release (LumieraFile self)
{
TRACE (file);
LUMIERA_MUTEX_SECTION (file, self->descriptor->rh, &self->descriptor->lock)
{
lumiera_filehandlecache_checkin (fhcache, self->descriptor->handle);
}
}

View file

@ -26,7 +26,8 @@
#include "lib/mutex.h"
#include "lib/llist.h"
#include "lib/error.h"
#include "lib/references.h"
NOBUG_DECLARE_FLAG (file);
/**
* @file File management
@ -47,18 +48,18 @@ typedef lumiera_file* LumieraFile;
#include "backend/filedescriptor.h"
NOBUG_DECLARE_FLAG (lumiera_file);
#define LUMIERA_FILE_READONLY (O_RDONLY | O_LARGEFILE | O_NOATIME)
#define LUMIERA_FILE_READWRITE (O_RDWR | O_LARGEFILE | O_NOATIME)
#define LUMIERA_FILE_CREATE (O_RDWR | O_LARGEFILE | O_NOATIME | O_CREAT | O_EXCL)
#define LUMIERA_FILE_RECREATE (O_RDWR | O_LARGEFILE | O_NOATIME | O_CREAT | O_TRUNC)
/* creat and excl flags will be masked out for descriptor lookup */
#define LUMIERA_FILE_MASK ~(O_CREAT | O_EXCL)
struct lumiera_file_struct
{
const char* name;
struct lumiera_reference_struct descriptor;
LumieraFiledescriptor descriptor;
};
/**
@ -86,5 +87,11 @@ void
lumiera_file_delete (LumieraFile self);
int
lumiera_file_handle_acquire (LumieraFile self);
void
lumiera_file_handle_release (LumieraFile self);
#endif

View file

@ -23,7 +23,10 @@
#include "lib/safeclib.h"
#include "lib/cuckoo.h"
#include "backend/file.h"
#include "backend/filedescriptor.h"
#include "backend/filehandle.h"
#include "backend/filehandlecache.h"
#include <sys/types.h>
#include <fcntl.h>
@ -141,6 +144,7 @@ lumiera_filedescriptor_acquire (const char* name, int flags)
}
int fd;
INFO (filedescriptor, "try creating file: %s", name);
fd = creat (name, 0777);
if (fd == -1)
{

70
src/backend/filehandle.c Normal file
View file

@ -0,0 +1,70 @@
/*
filehandle - filehandle management and caching
Copyright (C) Lumiera.org
2008, Christian Thaeter <ct@pipapo.org>
This program is free software; you can redistribute it and/or
modify it under the terms of the GNU General Public License as
published by the Free Software Foundation; either version 2 of the
License, or (at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program; if not, write to the Free Software
Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
*/
#include "lib/llist.h"
#include "lib/safeclib.h"
#include "backend/file.h"
#include "backend/filehandle.h"
#include "backend/filedescriptor.h"
#include <unistd.h>
NOBUG_DEFINE_FLAG_PARENT (filehandle, file_all);
LumieraFilehandle
lumiera_filehandle_new ()
{
LumieraFilehandle self = lumiera_malloc (sizeof (lumiera_filehandle));
TRACE (filehandle, "%p", self);
llist_init (&self->cachenode);
self->fd = -1;
self->use_cnt = 1;
self->descriptor = NULL;
return self;
}
void*
lumiera_filehandle_destroy_node (LList node)
{
TRACE (filehandle);
REQUIRE (llist_is_empty (node));
LumieraFilehandle self = (LumieraFilehandle)node;
REQUIRE (self->use_cnt == 0);
if (self->fd >= 0)
close (self->fd);
self->fd = -1;
self->descriptor = NULL;
return self;
}
int
lumiera_filehandle_get (LumieraFilehandle self)
{
REQUIRE (self->descriptor);
return -1;
}

View file

@ -24,6 +24,11 @@
#include "lib/error.h"
#include "lib/llist.h"
typedef struct lumiera_filehandle_struct lumiera_filehandle;
typedef lumiera_filehandle* LumieraFilehandle;
#include "backend/filedescriptor.h"
/**
* @file Filehandles
*/
@ -35,16 +40,17 @@
struct lumiera_filehandle_struct
{
llist cachenode;
int handle;
//LumieraFile file;
//lumiera_mutex lock;
int fd;
unsigned use_cnt;
LumieraFiledescriptor descriptor;
};
typedef struct lumiera_filehandle_struct lumiera_filehandle;
typedef lumiera_filehandle* LumieraFilehandle;
LumieraFilehandle
lumiera_filehandle_new ();
int
lumiera_filehandle_get (LumieraFilehandle self);
void*
lumiera_filehandle_destroy_node (LList node);

View file

@ -19,41 +19,60 @@
Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
*/
#include "lib/safeclib.h"
#include "backend/file.h"
#include "backend/filehandlecache.h"
NOBUG_DEFINE_FLAG (lumiera_filehandlecache);
NOBUG_DEFINE_FLAG_PARENT (filehandlecache, file_all);
/* errors */
LUMIERA_ERROR_DEFINE (FILEHANDLECACHE_NOHANDLE, "No filehandle available");
/**/
LumieraFilehandlecache fhcache = NULL;
LumieraFilehandlecache
lumiera_filehandlecache_init (LumieraFilehandlecache self, int max_entries)
void
lumiera_filehandlecache_new (int max_entries)
{
lumiera_mrucache_init (&self->cache, lumiera_filehandle_destroy_node);
self->available =max_entries;
lumiera_mutex_init (&self->lock);
return self;
if (!fhcache)
{
NOBUG_INIT_FLAG (filehandlecache);
fhcache = lumiera_malloc (sizeof (lumiera_filehandlecache));
lumiera_mrucache_init (&fhcache->cache, lumiera_filehandle_destroy_node);
fhcache->available = max_entries;
fhcache->checked_out = 0;
lumiera_mutex_init (&fhcache->lock);
RESOURCE_ANNOUNCE (filehandlecache, "mutex", "filehandlecache", fhcache, fhcache->rh);
}
else
NOTREACHED;
}
LumieraFilehandlecache
lumiera_filehandlecache_destroy (LumieraFilehandlecache self)
void
lumiera_filehandlecache_delete (void)
{
self->available += self->cache.cached;
lumiera_mrucache_destroy (&self->cache);
lumiera_mutex_destroy (&self->lock);
return self;
if (fhcache)
{
REQUIRE (!fhcache->checked_out, "Filehandles in use at shutdown");
RESOURCE_FORGET (filehandlecache, fhcache->rh);
lumiera_mrucache_destroy (&fhcache->cache);
lumiera_mutex_destroy (&fhcache->lock);
free (fhcache);
fhcache = NULL;
}
}
LumieraFilehandle
lumiera_filehandlecache_filehandle_acquire (LumieraFilehandlecache self)
lumiera_filehandlecache_handle_acquire (LumieraFilehandlecache self, LumieraFiledescriptor desc)
{
LumieraFilehandle ret;
LUMIERA_MUTEX_SECTION (&self->lock)
TRACE (filehandlecache);
LumieraFilehandle ret = NULL;
LUMIERA_MUTEX_SECTION (filehandlecache, self->rh, &self->lock)
{
if (self->available <= 0 && self->cache.cached)
{
@ -65,19 +84,58 @@ lumiera_filehandlecache_filehandle_acquire (LumieraFilehandlecache self)
}
else
{
/* allocate new filehandle if we are below the limit or no cached handles where available (overallocating) */
/* allocate new filehandle if we are below the limit or no cached handles are available (overallocating) */
ret = lumiera_filehandle_new ();
if (!ret)
LUMIERA_ERROR_SET (lumiera_filehandlecache, FILEHANDLECACHE_NOHANDLE);
LUMIERA_ERROR_SET (filehandlecache, FILEHANDLECACHE_NOHANDLE);
else
--self->available;
}
ret->use_cnt = 1;
ret->descriptor = desc;
desc->handle = ret;
++self->checked_out;
}
return ret;
}
void
lumiera_filehandlecache_add (LumieraFilehandlecache self, LumieraFilehandle handle)
LumieraFilehandle
lumiera_filehandlecache_checkout (LumieraFilehandlecache self, LumieraFilehandle handle)
{
UNIMPLEMENTED ("");
REQUIRE (self);
REQUIRE (handle);
/* This function is called with the associated descriptor locked, nothing can modify 'handle' */
if (!handle->use_cnt)
{
/* lock cache and checkout */
LUMIERA_MUTEX_SECTION (filehandlecache, self->rh, &self->lock)
{
lumiera_mrucache_checkout (&self->cache, &handle->cachenode);
}
}
++handle->use_cnt;
++self->checked_out;
return handle;
}
void
lumiera_filehandlecache_checkin (LumieraFilehandlecache self, LumieraFilehandle handle)
{
REQUIRE (self);
REQUIRE (handle);
REQUIRE (handle->use_cnt);
/* This function is called with the associated descriptor locked, nothing can modify 'self' */
if (!--handle->use_cnt)
{
/* lock cache and checin */
LUMIERA_MUTEX_SECTION (filehandlecache, self->rh, &self->lock)
{
--self->checked_out;
lumiera_mrucache_checkin (&self->cache, &handle->cachenode);
}
}
}

View file

@ -25,6 +25,9 @@
#include "lib/mrucache.h"
#include "lib/mutex.h"
typedef struct lumiera_filehandlecache_struct lumiera_filehandlecache;
typedef lumiera_filehandlecache* LumieraFilehandlecache;
#include "backend/filehandle.h"
/**
@ -43,55 +46,55 @@ struct lumiera_filehandlecache_struct
{
lumiera_mrucache cache;
int available;
int checked_out;
lumiera_mutex lock;
RESOURCE_HANDLE (rh);
};
typedef struct lumiera_filehandlecache_struct lumiera_filehandlecache;
typedef lumiera_filehandlecache* LumieraFilehandlecache;
extern LumieraFilehandlecache fhcache;
/**
* Initializes a filehandle cache.
* @param self pointer to the cache to be initialized
* Initializes the filehandle cache.
* @param max_entries number how much filehandles shall be managed
* @return self as given
* The number of elements the cache can hold is static and should be
* determined by sysconf (_SC_OPEN_MAX) minus some safety margin.
* determined by sysconf (_SC_OPEN_MAX) minus some (big) safety margin.
*/
LumieraFilehandlecache
lumiera_filehandlecache_init (LumieraFilehandlecache self, int max_entries);
void
lumiera_filehandlecache_new (int max_entries);
/**
* Destroy a filehandle cache.
* @param self the cache to be destroyed
* @return self as given to the now uninitialized! cache
* Delete the filehandle cache.
* No filehandles in the cache must be locked, this would be a fatal error.
* The handles are closed automatically.
*/
LumieraFilehandlecache
lumiera_filehandlecache_destroy (LumieraFilehandlecache self);
void
lumiera_filehandlecache_delete (void);
/**
* Get a fresh filehandle.
* @param self pointer to the cache
* @return the new filehandle
* one must call lumiera_mutexacquirer_unlock (&lock) to free the lock on the handle when done
*/
LumieraFilehandle
lumiera_filehandlecache_filehandle_acquire (LumieraFilehandlecache self);
lumiera_filehandlecache_handle_acquire (LumieraFilehandlecache self, LumieraFiledescriptor desc);
/**
* Add filehande back to cache.
* Add filehande back to cache, the filehandle becomes subject of aging.
* @param self pointer to the cache
* @param handle filehandle to be put back
*/
void
lumiera_filehandlecache_add (LumieraFilehandlecache self, LumieraFilehandle handle);
lumiera_filehandlecache_add_filehandle (LumieraFilehandlecache self, LumieraFilehandle handle);
/**
*
* Remove a filehandle from cache aging
* Filehandles which are subject of cache aging must be checked out before they can be used
*
*/
LumieraFilehandle
lumiera_filehandlecache_remove (LumieraFilehandlecache self, int fd);
lumiera_filehandlecache_checkout (LumieraFilehandlecache self, LumieraFilehandle handle);
void
lumiera_filehandlecache_checkin (LumieraFilehandlecache self, LumieraFilehandle handle);
#endif

7
tests/20filehandle.tests Normal file
View file

@ -0,0 +1,7 @@
TESTING "Filehandle management" ./test-filehandles
TEST "basic filehandle usage" basic <<END
return: 0
END

View file

@ -56,9 +56,9 @@ test_filedescriptors_SOURCES = $(tests_srcdir)/backend/test-filedescriptors.c
test_filedescriptors_CPPFLAGS = $(AM_CPPFLAGS) -std=gnu99 -Wall -Werror -I$(top_srcdir)/src/
test_filedescriptors_LDADD = liblumibackend.a liblumi.a -lnobugmt -lpthread -ldl -lm
#check_PROGRAMS += test-filehandles
#test_filehandles_SOURCES = $(tests_srcdir)/backend/test-filehandles.c
#test_filehandles_CPPFLAGS = $(AM_CPPFLAGS) -std=gnu99 -Wall -Werror -I$(top_srcdir)/src/
#test_filehandles_LDADD = liblumibackend.a liblumi.a -lnobugmt -lpthread -ldl -lm
check_PROGRAMS += test-filehandles
test_filehandles_SOURCES = $(tests_srcdir)/backend/test-filehandles.c
test_filehandles_CPPFLAGS = $(AM_CPPFLAGS) -std=gnu99 -Wall -Werror -I$(top_srcdir)/src/
test_filehandles_LDADD = liblumibackend.a liblumi.a -lnobugmt -lpthread -ldl -lm
TESTS = $(tests_srcdir)/test.sh

View file

@ -22,19 +22,21 @@
#include "lib/safeclib.h"
#include "backend/backend.h"
#include "backend/filedescriptor.h"
#include "tests/test.h"
TESTS_BEGIN
TEST ("acquire_existing")
{
lumiera_backend_init ();
LumieraFiledescriptor descriptor = lumiera_filedescriptor_acquire (",tmp_testfile", LUMIERA_FILE_READONLY);
if (descriptor)
{
lumiera_filedescriptor_release (descriptor);
lumiera_backend_destroy ();
return 0;
}
else
@ -43,12 +45,14 @@ TEST ("acquire_existing")
TEST ("acquire_existing_again")
{
lumiera_backend_init ();
LumieraFiledescriptor descriptor = lumiera_filedescriptor_acquire (",tmp_testfile", LUMIERA_FILE_READONLY);
if (descriptor)
{
LumieraFiledescriptor descriptor2 = lumiera_filedescriptor_acquire (",tmp_testfile", LUMIERA_FILE_READONLY);
lumiera_filedescriptor_release (descriptor2);
lumiera_filedescriptor_release (descriptor);
lumiera_backend_destroy ();
return 0;
}
else
@ -57,10 +61,12 @@ TEST ("acquire_existing_again")
TEST ("acquire_create")
{
lumiera_backend_init ();
LumieraFiledescriptor descriptor = lumiera_filedescriptor_acquire (",tmp_testfile", LUMIERA_FILE_CREATE);
if (descriptor)
{
lumiera_filedescriptor_release (descriptor);
lumiera_backend_destroy ();
return 0;
}
else
@ -69,11 +75,13 @@ TEST ("acquire_create")
TEST ("acquire_create_dir")
{
lumiera_backend_init ();
LumieraFiledescriptor descriptor =
lumiera_filedescriptor_acquire (",tmp_testdir/nested/,tmp_testfile", LUMIERA_FILE_CREATE);
if (descriptor)
{
lumiera_filedescriptor_release (descriptor);
lumiera_backend_destroy ();
return 0;
}
else

View file

@ -0,0 +1,58 @@
/*
test-filehandles.c - test filehandle management
Copyright (C) Lumiera.org
2008, Christian Thaeter <ct@pipapo.org>
This program is free software; you can redistribute it and/or
modify it under the terms of the GNU General Public License as
published by the Free Software Foundation; either version 2 of the
License, or (at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program; if not, write to the Free Software
Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
*/
#define _GNU_SOURCE
#include "lib/llist.h"
#include "backend/backend.h"
#include "backend/filehandlecache.h"
#include "tests/test.h"
TESTS_BEGIN
TEST ("basic")
{
lumiera_backend_init ();
LumieraFile file = lumiera_file_new (",tmp_testfile", LUMIERA_FILE_CREATE);
/* get the filehandle */
int fd = lumiera_file_handle_acquire (file);
/* we now 'own'it and can use it */
ENSURE (fd > -1);
printf ("got filehandle #%d\n", fd);
/* put it into aging, can't use anymore */
lumiera_file_handle_release (file);
lumiera_file_delete (file);
lumiera_backend_destroy ();
}
TEST ("more")
{
lumiera_backend_init ();
lumiera_backend_destroy ();
}
TESTS_END