WIP: Low level file management

We have 'File' which associates a name with a file and will be used to
offer an interface for the application.

Posix 'Filehandles' which are managed in a 'Filehandlecache' to utilize
a limited number of fd's at optimum.

Internally we use 'Filedescriptor's which manage files, mapping and all
kinds of metadata. Filedescriptors weak referenced by a registry and freed
when the last user vanishes.
This commit is contained in:
Christian Thaeter 2008-03-30 04:31:32 +02:00
parent 94e075d2c2
commit 791f42c104
8 changed files with 649 additions and 8 deletions

View file

@ -31,12 +31,11 @@ liblumibackend_a_SOURCES = \
# $(liblumibackend_a_srcdir)/filehandlecache.c
noinst_HEADERS += \
$(liblumibackend_a_srcdir)/mediaaccessfacade.cpp
# $(liblumibackend_a_srcdir)/backend.h \
# $(liblumibackend_a_srcdir)/file.h \
# $(liblumibackend_a_srcdir)/filehandle.h \
# $(liblumibackend_a_srcdir)/filedescriptor.h \
# $(liblumibackend_a_srcdir)/filehandlecache.h
noinst_HEADERS += \
$(liblumibackend_a_srcdir)/mediaaccessfacade.cpp \
$(liblumibackend_a_srcdir)/backend.h \
$(liblumibackend_a_srcdir)/file.h \
$(liblumibackend_a_srcdir)/filehandle.h \
$(liblumibackend_a_srcdir)/filedescriptor.h \
$(liblumibackend_a_srcdir)/filehandlecache.h

64
src/backend/file.c Normal file
View file

@ -0,0 +1,64 @@
/*
file.c - file handling
Copyright (C) Lumiera.org
2008, Christian Thaeter <ct@pipapo.org>
This program is free software; you can redistribute it and/or
modify it under the terms of the GNU General Public License as
published by the Free Software Foundation; either version 2 of the
License, or (at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program; if not, write to the Free Software
Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
*/
#include "lib/mutex.h"
#include "lib/safeclib.h"
#include "backend/file.h"
#include "backend/filedescriptor.h"
#include <limits.h>
NOBUG_DEFINE_FLAG (lumiera_file);
LumieraFile
lumiera_file_init (LumieraFile self, const char* name, int flags)
{
if (!lumiera_filedescriptor_acquire (&self->descriptor, name, flags))
return NULL;
self->name = lumiera_strndup (name, PATH_MAX);
return self;
}
LumieraFile
lumiera_file_destroy (LumieraFile self)
{
lumiera_reference_destroy (&self->descriptor);
free ((void*)self->name);
return self;
}
LumieraFile
lumiera_file_new (const char* name, int flags)
{
LumieraFile self = lumiera_malloc (sizeof (lumiera_file));
return lumiera_file_init (self, name, flags);
}
void
lumiera_file_delete (LumieraFile self)
{
free (lumiera_file_destroy (self));
}

90
src/backend/file.h Normal file
View file

@ -0,0 +1,90 @@
/*
file.h - interface to files by filename
Copyright (C) Lumiera.org
2008, Christian Thaeter <ct@pipapo.org>
This program is free software; you can redistribute it and/or
modify it under the terms of the GNU General Public License as
published by the Free Software Foundation; either version 2 of the
License, or (at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program; if not, write to the Free Software
Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
*/
#ifndef LUMIERA_FILE_H
#define LUMIERA_FILE_H
#include "lib/mutex.h"
#include "lib/llist.h"
#include "lib/error.h"
#include "lib/references.h"
/**
* @file File management
* Handling Files is splitted into different classes:
* 1. The 'lumiera_file' class which acts as interface to the outside for managing files.
* 'lumiera_file' is addressed by the name of the file. Since files can have more than one name (hardlinks)
* many 'lumiera_file' can point to a single 'lumiera_filedescriptor'
* 2. The 'lumiera_filedescriptor' class which does the real work managing the file in the back.
* 3. Since OS-filehandles are a limited resource we access the lazily as 'lumiera_filehandle' which are
* managed in a 'lumiera_filehandlecache'
*/
typedef struct lumiera_file_struct lumiera_file;
typedef lumiera_file* LumieraFile;
#include "backend/filehandle.h"
#include "backend/filedescriptor.h"
NOBUG_DECLARE_FLAG (lumiera_file);
#define LUMIERA_FILE_READONLY (O_RDONLY | O_LARGEFILE | O_NOATIME)
#define LUMIERA_FILE_READWRITE (O_RDWR | O_LARGEFILE | O_NOATIME)
#define LUMIERA_FILE_CREATE (O_RDWR | O_LARGEFILE | O_NOATIME | O_CREAT | O_EXCL)
#define LUMIERA_FILE_RECREATE (O_RDWR | O_LARGEFILE | O_NOATIME | O_CREAT | O_TRUNC)
struct lumiera_file_struct
{
const char* name;
struct lumiera_reference_struct descriptor;
};
/**
*
*/
LumieraFile
lumiera_file_init (LumieraFile self, const char* name, int flags);
/**
*
*/
LumieraFile
lumiera_file_destroy (LumieraFile self);
/**
*
*/
LumieraFile
lumiera_file_new (const char* name, int flags);
/**
*
*/
void
lumiera_file_delete (LumieraFile self);
#endif

View file

@ -0,0 +1,199 @@
/*
filedescriptor.c - file handling
Copyright (C) Lumiera.org
2008, Christian Thaeter <ct@pipapo.org>
This program is free software; you can redistribute it and/or
modify it under the terms of the GNU General Public License as
published by the Free Software Foundation; either version 2 of the
License, or (at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program; if not, write to the Free Software
Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
*/
#include "lib/mutex.h"
#include "lib/safeclib.h"
#include "lib/cuckoo.h"
#include "backend/filedescriptor.h"
#include <sys/types.h>
#include <fcntl.h>
#include <limits.h>
#include <unistd.h>
NOBUG_DEFINE_FLAG (filedescriptor);
static Cuckoo registry = NULL;
static lumiera_mutex registry_mutex = {PTHREAD_MUTEX_INITIALIZER};
static size_t
h1 (const void* item, const uint32_t r)
{
const LumieraFiledescriptor i = (const LumieraFiledescriptor) item;
return i->stat.st_dev^i->stat.st_ino^i->flags^((i->stat.st_dev^i->stat.st_ino^i->flags)>>7)^r;
}
static size_t
h2 (const void* item, const uint32_t r)
{
const LumieraFiledescriptor i = (const LumieraFiledescriptor) item;
return i->stat.st_dev^i->stat.st_ino^i->flags^((i->stat.st_dev^i->stat.st_ino^i->flags)>>5)^r;
}
static size_t
h3 (const void* item, const uint32_t r)
{
const LumieraFiledescriptor i = (const LumieraFiledescriptor) item;
return i->stat.st_dev^i->stat.st_ino^i->flags^((i->stat.st_dev^i->stat.st_ino^i->flags)>>3)^r;
}
static int
cmp (const void* keya, const void* keyb)
{
const LumieraFiledescriptor a = (const LumieraFiledescriptor) keya;
const LumieraFiledescriptor b = (const LumieraFiledescriptor) keyb;
return a->stat.st_dev == b->stat.st_dev && a->stat.st_ino == b->stat.st_ino && a->flags == b->flags;
}
static Cuckoo
lumiera_filedescriptor_registry_new (void)
{
NOBUG_INIT_FLAG (filedescriptor);
Cuckoo registry = cuckoo_new (h1, h2, h3, cmp,
sizeof (struct lumiera_reference_struct),
3);
if (!registry)
LUMIERA_DIE (NO_MEMORY);
return registry;
}
/**
* Find existing filedescriptor or create one
* @param descriptor strong reference to be initialized with the filedescriptor,
* a filedescriptor will be deleted when its last string reference gets deleted.
* @param name name of the file
* @param flags opening flags for the filedescriptor
* @return descriptor on success or NULL on error
*/
LumieraReference
lumiera_filedescriptor_acquire (LumieraReference descriptor, const char* name, int flags)
{
UNCHECKED;
lumiera_mutexacquirer registry_lock;
lumiera_mutexacquirer_init_mutex (&registry_lock, &registry_mutex, LUMIERA_LOCKED);
if (!registry)
registry = lumiera_filedescriptor_registry_new ();
struct lumiera_filedescriptor_struct findme;
findme.flags = flags;
for (int retry = 0; !retry; ++retry)
{
if (stat (name, &findme.stat) == 0)
break;
else
{
if (errno == ENOENT && flags&O_CREAT)
{
TRACE (filedescriptor, "try creating file: %s", name);
char* slash = lumiera_tmpbuf_strndup (name, PATH_MAX);
while ((slash = strchr (slash+1, '/')))
{
*slash = '\0';
if (mkdir (name, 0777) == -1)
{
LUMIERA_ERROR_SET (filedescriptor, ERRNO);
goto efile;
}
*slash = '/';
}
int fd;
fd = creat (name, 0777);
if (fd == -1)
{
LUMIERA_ERROR_SET (filedescriptor, ERRNO);
goto efile;
}
close (fd);
/* will now retry the fstat once */
}
else
{
LUMIERA_ERROR_SET (filedescriptor, ERRNO);
goto efile;
}
}
}
/* lookup/create descriptor */
LumieraReference elem = cuckoo_find (registry, &findme);
if (!elem)
{
TRACE (filedescriptor, "Descriptor not found");
lumiera_reference_strong_init_once (descriptor,
lumiera_filedescriptor_new (&findme),
(void(*)(void*))lumiera_filedescriptor_delete);
if (!lumiera_reference_get (descriptor))
goto ecreate;
lumiera_reference weak_descriptor;
lumiera_reference_weak_init (&weak_descriptor, descriptor);
cuckoo_insert (registry, &weak_descriptor);
}
else
{
lumiera_reference_strong_init (descriptor, elem);
}
lumiera_mutexacquirer_unlock (&registry_lock);
return descriptor;
efile:
ecreate:
lumiera_mutexacquirer_unlock (&registry_lock);
return NULL;
}
/**
* Allocate a new filedescriptor cloned from a template
* @param template the source filedescriptor
* @return the constrccted filedescriptor
*/
LumieraFiledescriptor
lumiera_filedescriptor_new (LumieraFiledescriptor template)
{
UNIMPLEMENTED("");
return NULL;
}
/**
* delete a filedescriptor and free its resources
* @param descriptor filedescriptor to be freed
*/
void
lumiera_filedescriptor_delete (LumieraFiledescriptor descriptor)
{
UNIMPLEMENTED("");
}

View file

@ -0,0 +1,58 @@
/*
filedescriptor.h - file handling
Copyright (C) Lumiera.org
2008, Christian Thaeter <ct@pipapo.org>
This program is free software; you can redistribute it and/or
modify it under the terms of the GNU General Public License as
published by the Free Software Foundation; either version 2 of the
License, or (at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program; if not, write to the Free Software
Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
*/
#ifndef LUMIERA_FILEDESCRIPTOR_H
#define LUMIERA_FILEDESCRIPTOR_H
#include "lib/mutex.h"
#include <sys/stat.h>
typedef struct lumiera_filedescriptor_struct lumiera_filedescriptor;
typedef lumiera_filedescriptor* LumieraFiledescriptor;
#include "backend/filehandle.h"
#include "backend/file.h"
struct lumiera_filedescriptor_struct
{
struct stat stat; /* create after first open, maintained metadata, MUST BE FIRST! */
int flags; /* open flags, must be masked for reopen */
lumiera_mutex lock; /* locks operations on this file descriptor */
unsigned reopened; /* count for reopens 0=not yet opened, 1=first, 2..=reopened*/
//LumieraFileMap mappings;
//LumieraWriteBuffer writebuffer;
};
LumieraReference
lumiera_filedescriptor_acquire (LumieraReference descriptor, const char* name, int flags);
LumieraFiledescriptor
lumiera_filedescriptor_new (LumieraFiledescriptor template);
void
lumiera_filedescriptor_delete (LumieraFiledescriptor self);
#endif

51
src/backend/filehandle.h Normal file
View file

@ -0,0 +1,51 @@
/*
filehandle - filehandle management and caching
Copyright (C) Lumiera.org
2008, Christian Thaeter <ct@pipapo.org>
This program is free software; you can redistribute it and/or
modify it under the terms of the GNU General Public License as
published by the Free Software Foundation; either version 2 of the
License, or (at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program; if not, write to the Free Software
Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
*/
#ifndef LUMIERA_FILEHANDLE_H
#define LUMIERA_FILEHANDLE_H
#include "lib/error.h"
#include "lib/llist.h"
/**
* @file Filehandles
*/
/**
* File handles
*/
struct lumiera_filehandle_struct
{
llist cachenode;
int handle;
//LumieraFile file;
//lumiera_mutex lock;
};
typedef struct lumiera_filehandle_struct lumiera_filehandle;
typedef lumiera_filehandle* LumieraFilehandle;
LumieraFilehandle
lumiera_filehandle_new ();
void*
lumiera_filehandle_destroy_node (LList node);
#endif

View file

@ -0,0 +1,83 @@
/*
filehandlecache - filehandle management and caching
Copyright (C) Lumiera.org
2008, Christian Thaeter <ct@pipapo.org>
This program is free software; you can redistribute it and/or
modify it under the terms of the GNU General Public License as
published by the Free Software Foundation; either version 2 of the
License, or (at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program; if not, write to the Free Software
Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
*/
#include "backend/filehandlecache.h"
NOBUG_DEFINE_FLAG (lumiera_filehandlecache);
/* errors */
LUMIERA_ERROR_DEFINE (FILEHANDLECACHE_NOHANDLE, "No filehandle available");
/**/
LumieraFilehandlecache
lumiera_filehandlecache_init (LumieraFilehandlecache self, int max_entries)
{
lumiera_mrucache_init (&self->cache, lumiera_filehandle_destroy_node);
self->available =max_entries;
lumiera_mutex_init (&self->lock);
return self;
}
LumieraFilehandlecache
lumiera_filehandlecache_destroy (LumieraFilehandlecache self)
{
self->available += self->cache.cached;
lumiera_mrucache_destroy (&self->cache);
lumiera_mutex_destroy (&self->lock);
return self;
}
LumieraFilehandle
lumiera_filehandlecache_filehandle_acquire (LumieraFilehandlecache self)
{
LumieraFilehandle ret;
LUMIERA_MUTEX_SECTION (&self->lock)
{
if (self->available <= 0 && self->cache.cached)
{
/* pop a filehandle from cache */
ret = lumiera_mrucache_pop (&self->cache);
if (self->available < 0)
/* try to free overallocated filehandles */
self->available -= self->available + lumiera_mrucache_age (&self->cache, -self->available);
}
else
{
/* allocate new filehandle if we are below the limit or no cached handles where available (overallocating) */
ret = lumiera_filehandle_new ();
if (!ret)
LUMIERA_ERROR_SET (lumiera_filehandlecache, FILEHANDLECACHE_NOHANDLE);
else
--self->available;
}
}
return ret;
}
void
lumiera_filehandlecache_add (LumieraFilehandlecache self, LumieraFilehandle handle)
{
UNIMPLEMENTED ("");
}

View file

@ -0,0 +1,97 @@
/*
filehandlecache - filehandle management and caching
Copyright (C) Lumiera.org
2008, Christian Thaeter <ct@pipapo.org>
This program is free software; you can redistribute it and/or
modify it under the terms of the GNU General Public License as
published by the Free Software Foundation; either version 2 of the
License, or (at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program; if not, write to the Free Software
Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
*/
#ifndef LUMIERA_FILEHANDLECACHE_H
#define LUMIERA_FILEHANDLECACHE_H
#include "lib/error.h"
#include "lib/mrucache.h"
#include "lib/mutex.h"
#include "backend/filehandle.h"
/**
* @file Filehandle management and caching
* The number of filehandles a program can held open is usually limited, since we want to support
* using a less limited number of files and closing/opening for each operation is expensive, we
* provide a cache to keep the most frequent used files open and gracefully close/recycle unused filehandles.
*/
/*
File handle cache manages file handles
*/
struct lumiera_filehandlecache_struct
{
lumiera_mrucache cache;
int available;
lumiera_mutex lock;
};
typedef struct lumiera_filehandlecache_struct lumiera_filehandlecache;
typedef lumiera_filehandlecache* LumieraFilehandlecache;
/**
* Initializes a filehandle cache.
* @param self pointer to the cache to be initialized
* @param max_entries number how much filehandles shall be managed
* @return self as given
* The number of elements the cache can hold is static and should be
* determined by sysconf (_SC_OPEN_MAX) minus some safety margin.
*/
LumieraFilehandlecache
lumiera_filehandlecache_init (LumieraFilehandlecache self, int max_entries);
/**
* Destroy a filehandle cache.
* @param self the cache to be destroyed
* @return self as given to the now uninitialized! cache
* No filehandles in the cache must be locked, this would be a fatal error.
* The handles are closed automatically.
*/
LumieraFilehandlecache
lumiera_filehandlecache_destroy (LumieraFilehandlecache self);
/**
* Get a fresh filehandle.
* @param self pointer to the cache
* @return the new filehandle
* one must call lumiera_mutexacquirer_unlock (&lock) to free the lock on the handle when done
*/
LumieraFilehandle
lumiera_filehandlecache_filehandle_acquire (LumieraFilehandlecache self);
/**
* Add filehande back to cache.
* @param self pointer to the cache
* @param handle filehandle to be put back
*/
void
lumiera_filehandlecache_add (LumieraFilehandlecache self, LumieraFilehandle handle);
/**
*
*/
LumieraFilehandle
lumiera_filehandlecache_remove (LumieraFilehandlecache self, int fd);
#endif