From 2d11d5186082aea7b6c818d28700ccafa1268834 Mon Sep 17 00:00:00 2001 From: Christian Thaeter Date: Fri, 16 May 2008 20:17:43 +0200 Subject: [PATCH 01/39] Checks for NoBug (and other) headers have to follow the particular configurations. If not the generated conftest would bail out NoBug complaining that no build level is selected --- configure.ac | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/configure.ac b/configure.ac index 78d76c136..72c616a17 100644 --- a/configure.ac +++ b/configure.ac @@ -37,9 +37,6 @@ AC_PROG_LIBTOOL # test for headers # AC_STDC_HEADERS -AC_CHECK_HEADER([nobug.h], AC_DEFINE(HAVE_NOBUG_H)) -AC_CHECK_HEADER([execinfo.h], AC_DEFINE(HAVE_EXECINFO_H)) -# there is a warning in nobug, disabled til fixed AC_CHECK_HEADER([valgrind/valgrind.h], AC_DEFINE(HAVE_VALGRIND_VALGRIND_H)) ACX_PTHREAD ##AM_CONDITIONAL([HAVE_THREADING], [test x"$acx_pthread_ok" = xyes]) @@ -71,6 +68,12 @@ AC_ARG_ENABLE(release, AC_HELP_STRING([--enable-release], [select NoBug RELEASE nobug_level=alpha AC_DEFINE(EBUG_ALPHA) )])]) + +AC_CHECK_HEADER([nobug.h], AC_DEFINE(HAVE_NOBUG_H)) +AC_CHECK_HEADER([execinfo.h], AC_DEFINE(HAVE_EXECINFO_H)) +# there is a warning in nobug, disabled til fixed AC_CHECK_HEADER([valgrind/valgrind.h], AC_DEFINE(HAVE_VALGRIND_VALGRIND_H)) + + AC_MSG_RESULT([NoBug build level: $nobug_level]) # END NoBug From 19fd89d13ddd392e6c244c2a3889f361896317bf Mon Sep 17 00:00:00 2001 From: Christian Thaeter Date: Sat, 17 May 2008 20:00:05 +0200 Subject: [PATCH 02/39] cleanup and add more configure checks * GNU_SOURCE by default now * improved use of ACX_PTHREAD, replaces $CC and others * all checks for mandatory things instantly exit for now in future this might produce an report at the end * ported checks from scons over here --- configure.ac | 59 +++++++++++++++++++++++++++++++++++++++++++++++++--- 1 file changed, 56 insertions(+), 3 deletions(-) diff --git a/configure.ac b/configure.ac index 72c616a17..b72b7dfee 100644 --- a/configure.ac +++ b/configure.ac @@ -23,6 +23,8 @@ AC_COPYRIGHT([ Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA. ]) +AC_GNU_SOURCE + # # Required programs # @@ -39,6 +41,10 @@ AC_PROG_LIBTOOL AC_STDC_HEADERS ACX_PTHREAD +LIBS="$PTHREAD_LIBS $LIBS" +CFLAGS="$CFLAGS $PTHREAD_CFLAGS" +CC="$PTHREAD_CC" + ##AM_CONDITIONAL([HAVE_THREADING], [test x"$acx_pthread_ok" = xyes]) @@ -68,15 +74,62 @@ AC_ARG_ENABLE(release, AC_HELP_STRING([--enable-release], [select NoBug RELEASE nobug_level=alpha AC_DEFINE(EBUG_ALPHA) )])]) +# END NoBug + +# C headers and libraries +AC_LANG_PUSH([C]) + +AC_CHECK_HEADER([nobug.h], + AC_DEFINE(HAVE_NOBUG_H), + AC_MSG_ERROR([NoBug header missing (http://www.pipapo.org/pipawiki/NoBug)]) + ) +AC_CHECK_LIB([nobugmt], nobug_thread_id_get, + , + AC_MSG_ERROR([NoBug multithreading library missing (http://www.pipapo.org/pipawiki/NoBug)]) + ) -AC_CHECK_HEADER([nobug.h], AC_DEFINE(HAVE_NOBUG_H)) AC_CHECK_HEADER([execinfo.h], AC_DEFINE(HAVE_EXECINFO_H)) # there is a warning in nobug, disabled til fixed AC_CHECK_HEADER([valgrind/valgrind.h], AC_DEFINE(HAVE_VALGRIND_VALGRIND_H)) +AC_LANG_POP([C]) -AC_MSG_RESULT([NoBug build level: $nobug_level]) -# END NoBug +# C++ headers and libraries +AC_LANG_PUSH([C++]) + +AC_CHECK_HEADER([tr1/memory], + , + AC_MSG_ERROR([std::tr1 proposed standard extension for shared_ptr missing]) + ) + +AC_CHECK_HEADER([boost/shared_ptr.hpp], + , + AC_MSG_ERROR([boost::shared_ptr missing (http://www.boost.org/)]) + ) + +AC_CHECK_HEADER([boost/program_options.hpp], + , + AC_MSG_ERROR([boost::program_options missing (http://www.boost.org/)]) + ) +# checking for library left out, name contains compiler version here (libboost_program_options-gcc42-mt-1_34_1.so.1.34.1) any ides how to use that? + +AC_CHECK_HEADER([boost/regex.hpp], + , + AC_MSG_ERROR([boost::regex missing (http://www.boost.org/)]) + ) + +AC_LANG_POP([C++]) + + +# Print a summary +AC_MSG_RESULT([ +Configuration Summary:]) +AC_MSG_RESULT([ NoBug build level: $nobug_level]) +# Add more summary results here + +AC_MSG_RESULT([ +Configuration complete, you can now build Lumiera with 'make' +]) AC_CONFIG_FILES(Makefile) AC_OUTPUT From 2cbfeda4df2ab5de06d8094124a52fccd53f10cc Mon Sep 17 00:00:00 2001 From: Christian Thaeter Date: Thu, 10 Apr 2008 06:20:51 +0200 Subject: [PATCH 03/39] correct return types in condition.h --- src/lib/condition.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/lib/condition.h b/src/lib/condition.h index 066c6b267..ffc00523c 100644 --- a/src/lib/condition.h +++ b/src/lib/condition.h @@ -167,7 +167,7 @@ lumiera_conditionacquirer_wait (LumieraConditionacquirer self) * a conditionacquirer must be unlocked before leaving scope * @param self conditionacquirer associated with a condition variable */ -static inline int +static inline void lumiera_conditionacquirer_unlock (LumieraConditionacquirer self) { REQUIRE (self); @@ -195,7 +195,7 @@ lumiera_conditionacquirer_signal (LumieraConditionacquirer self) * signal all waiting threads * @param self conditionacquirer associated with the condition variable to be signaled */ -static inline int +static inline void lumiera_conditionacquirer_broadcast (LumieraConditionacquirer self) { REQUIRE (self); From 249c6d5eda5c8ccbf4bc9f575c80b048812e6954 Mon Sep 17 00:00:00 2001 From: Christian Thaeter Date: Sat, 17 May 2008 23:33:21 +0200 Subject: [PATCH 04/39] Remove _GNU_SOURCE from certain files GNU_SOURCE is globally activated in configure.ac --- admin/vgsuppression.c | 1 - src/lib/rwlock.c | 2 -- src/lib/safeclib.c | 2 -- 3 files changed, 5 deletions(-) diff --git a/admin/vgsuppression.c b/admin/vgsuppression.c index 0e1f41fca..cad026e52 100644 --- a/admin/vgsuppression.c +++ b/admin/vgsuppression.c @@ -18,7 +18,6 @@ along with this program; if not, write to the Free Software Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA. */ -#define _GNU_SOURCE /* just place any problematic calls where valgrind whines about in main (with comments please) diff --git a/src/lib/rwlock.c b/src/lib/rwlock.c index 0862aa808..092c7b54b 100644 --- a/src/lib/rwlock.c +++ b/src/lib/rwlock.c @@ -18,8 +18,6 @@ along with this program; if not, write to the Free Software Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA. */ -#define _GNU_SOURCE - #include #include "lib/rwlock.h" diff --git a/src/lib/safeclib.c b/src/lib/safeclib.c index 71e347ab7..a0b562643 100644 --- a/src/lib/safeclib.c +++ b/src/lib/safeclib.c @@ -18,8 +18,6 @@ along with this program; if not, write to the Free Software Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA. */ -#define _GNU_SOURCE - #include "error.h" #include From e8164062a29d8e68951678f58ae8935637471c59 Mon Sep 17 00:00:00 2001 From: Christian Thaeter Date: Sun, 18 May 2008 00:01:51 +0200 Subject: [PATCH 05/39] integrate proc into automake * all files in src/proc are build into libs * TODO: -Werror still missing * TODO: Tests are not yet compiled --- Makefile.am | 1 + src/proc/Makefile.am | 263 +++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 264 insertions(+) create mode 100644 src/proc/Makefile.am diff --git a/Makefile.am b/Makefile.am index 69cc34288..eb67e0a23 100644 --- a/Makefile.am +++ b/Makefile.am @@ -33,6 +33,7 @@ SUBDIRS = include $(top_srcdir)/admin/Makefile.am # core +include $(top_srcdir)/src/proc/Makefile.am include $(top_srcdir)/src/lib/Makefile.am #include $(top_srcdir)/src/backend/Makefile.am diff --git a/src/proc/Makefile.am b/src/proc/Makefile.am new file mode 100644 index 000000000..612ffc2d6 --- /dev/null +++ b/src/proc/Makefile.am @@ -0,0 +1,263 @@ +# Copyright (C) Lumiera.org +# 2008, Christian Thaeter +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License as +# published by the Free Software Foundation; either version 2 of the +# License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# Copyright (C) Lumiera.org +# 2008, Christian Thaeter +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License as +# published by the Free Software Foundation; either version 2 of the +# License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA. + + +# TODO use -Wextra -Werror + +liblumiproc_a_srcdir = $(top_srcdir)/src/proc +noinst_LIBRARIES += liblumiproc.a + +liblumiproc_a_CXXFLAGS = $(CXXFLAGS) -Wall +liblumiproc_a_CPPFLAGS = -I$(top_srcdir)/src/ + +liblumiproc_a_SOURCES = \ + $(liblumiproc_a_srcdir)/controllerfacade.cpp \ + $(liblumiproc_a_srcdir)/frame.cpp \ + $(liblumiproc_a_srcdir)/stateproxy.cpp \ + $(liblumiproc_a_srcdir)/asset.cpp \ + $(liblumiproc_a_srcdir)/assetmanager.cpp \ + $(liblumiproc_a_srcdir)/nobugcfg.cpp + + + +liblumiprocasset_a_srcdir = $(top_srcdir)/src/proc/asset +noinst_LIBRARIES += liblumiprocasset.a + +liblumiprocasset_a_CXXFLAGS = $(CXXFLAGS) -Wall +liblumiprocasset_a_CPPFLAGS = -I$(top_srcdir)/src/ + +liblumiprocasset_a_SOURCES = \ + $(liblumiprocasset_a_srcdir)/codec.cpp \ + $(liblumiprocasset_a_srcdir)/compoundmedia.cpp \ + $(liblumiprocasset_a_srcdir)/dataset.cpp \ + $(liblumiprocasset_a_srcdir)/effect.cpp \ + $(liblumiprocasset_a_srcdir)/preview.cpp \ + $(liblumiprocasset_a_srcdir)/unknown.cpp \ + $(liblumiprocasset_a_srcdir)/clip.cpp \ + $(liblumiprocasset_a_srcdir)/meta.cpp \ + $(liblumiprocasset_a_srcdir)/category.cpp \ + $(liblumiprocasset_a_srcdir)/media.cpp \ + $(liblumiprocasset_a_srcdir)/pipe.cpp \ + $(liblumiprocasset_a_srcdir)/proc.cpp \ + $(liblumiprocasset_a_srcdir)/procpatt.cpp \ + $(liblumiprocasset_a_srcdir)/struct.cpp \ + $(liblumiprocasset_a_srcdir)/track.cpp + + + +liblumiprocengine_a_srcdir = $(top_srcdir)/src/proc/engine +noinst_LIBRARIES += liblumiprocengine.a + +liblumiprocengine_a_CXXFLAGS = $(CXXFLAGS) -Wall +liblumiprocengine_a_CPPFLAGS = -I$(top_srcdir)/src/ + +liblumiprocengine_a_SOURCES = \ + $(liblumiprocengine_a_srcdir)/aframe.cpp \ + $(liblumiprocengine_a_srcdir)/arender.cpp \ + $(liblumiprocengine_a_srcdir)/codecadapter.cpp \ + $(liblumiprocengine_a_srcdir)/exitnode.cpp \ + $(liblumiprocengine_a_srcdir)/glbuf.cpp \ + $(liblumiprocengine_a_srcdir)/glpipe.cpp \ + $(liblumiprocengine_a_srcdir)/glrender.cpp \ + $(liblumiprocengine_a_srcdir)/hub.cpp \ + $(liblumiprocengine_a_srcdir)/link.cpp \ + $(liblumiprocengine_a_srcdir)/mask.cpp \ + $(liblumiprocengine_a_srcdir)/pluginadapter.cpp \ + $(liblumiprocengine_a_srcdir)/processor.cpp \ + $(liblumiprocengine_a_srcdir)/procnode.cpp \ + $(liblumiprocengine_a_srcdir)/projector.cpp \ + $(liblumiprocengine_a_srcdir)/renderengine.cpp \ + $(liblumiprocengine_a_srcdir)/source.cpp \ + $(liblumiprocengine_a_srcdir)/trafo.cpp \ + $(liblumiprocengine_a_srcdir)/vframe.cpp \ + $(liblumiprocengine_a_srcdir)/vrender.cpp + + + +liblumiprocmobject_a_srcdir = $(top_srcdir)/src/proc/mobject +noinst_LIBRARIES += liblumiprocmobject.a + +liblumiprocmobject_a_CXXFLAGS = $(CXXFLAGS) -Wall +liblumiprocmobject_a_CPPFLAGS = -I$(top_srcdir)/src/ + +liblumiprocmobject_a_SOURCES = \ + $(liblumiprocmobject_a_srcdir)/builderfacade.cpp \ + $(liblumiprocmobject_a_srcdir)/interpolator.cpp \ + $(liblumiprocmobject_a_srcdir)/mobject.cpp \ + $(liblumiprocmobject_a_srcdir)/parameter.cpp \ + $(liblumiprocmobject_a_srcdir)/paramprovider.cpp \ + $(liblumiprocmobject_a_srcdir)/placement.cpp + + + +liblumiprocmobjectbuilder_a_srcdir = $(top_srcdir)/src/proc/mobject/builder +noinst_LIBRARIES += liblumiprocmobjectbuilder.a + +liblumiprocmobjectbuilder_a_CXXFLAGS = $(CXXFLAGS) -Wall +liblumiprocmobjectbuilder_a_CPPFLAGS = -I$(top_srcdir)/src/ + +liblumiprocmobjectbuilder_a_SOURCES = \ + $(liblumiprocmobjectbuilder_a_srcdir)/assembler.cpp \ + $(liblumiprocmobjectbuilder_a_srcdir)/conmanager.cpp \ + $(liblumiprocmobjectbuilder_a_srcdir)/nodecreatertool.cpp \ + $(liblumiprocmobjectbuilder_a_srcdir)/segmentationtool.cpp \ + $(liblumiprocmobjectbuilder_a_srcdir)/toolfactory.cpp + + +liblumiprocmobjectcontroller_a_srcdir = $(top_srcdir)/src/proc/mobject/controller +noinst_LIBRARIES += liblumiprocmobjectcontroller.a + +liblumiprocmobjectcontroller_a_CXXFLAGS = $(CXXFLAGS) -Wall +liblumiprocmobjectcontroller_a_CPPFLAGS = -I$(top_srcdir)/src/ + +liblumiprocmobjectcontroller_a_SOURCES = \ + $(liblumiprocmobjectcontroller_a_srcdir)/pathmanager.cpp \ + $(liblumiprocmobjectcontroller_a_srcdir)/renderstate.cpp + + +liblumiprocmobjectsession_a_srcdir = $(top_srcdir)/src/proc/mobject/session +noinst_LIBRARIES += liblumiprocmobjectsession.a + +liblumiprocmobjectsession_a_CXXFLAGS = $(CXXFLAGS) -Wall +liblumiprocmobjectsession_a_CPPFLAGS = -I$(top_srcdir)/src/ + +liblumiprocmobjectsession_a_SOURCES = \ + $(liblumiprocmobjectsession_a_srcdir)/abstractmo.cpp \ + $(liblumiprocmobjectsession_a_srcdir)/allocation.cpp \ + $(liblumiprocmobjectsession_a_srcdir)/auto.cpp \ + $(liblumiprocmobjectsession_a_srcdir)/compoundclip.cpp \ + $(liblumiprocmobjectsession_a_srcdir)/effect.cpp \ + $(liblumiprocmobjectsession_a_srcdir)/fixedlocation.cpp \ + $(liblumiprocmobjectsession_a_srcdir)/label.cpp \ + $(liblumiprocmobjectsession_a_srcdir)/meta.cpp \ + $(liblumiprocmobjectsession_a_srcdir)/relativelocation.cpp \ + $(liblumiprocmobjectsession_a_srcdir)/segment.cpp \ + $(liblumiprocmobjectsession_a_srcdir)/simpleclip.cpp \ + $(liblumiprocmobjectsession_a_srcdir)/plug.cpp \ + $(liblumiprocmobjectsession_a_srcdir)/sessionimpl.cpp \ + $(liblumiprocmobjectsession_a_srcdir)/sessmanagerimpl.cpp \ + $(liblumiprocmobjectsession_a_srcdir)/edl.cpp \ + $(liblumiprocmobjectsession_a_srcdir)/session.cpp \ + $(liblumiprocmobjectsession_a_srcdir)/track.cpp \ + $(liblumiprocmobjectsession_a_srcdir)/clip.cpp \ + $(liblumiprocmobjectsession_a_srcdir)/constraint.cpp \ + $(liblumiprocmobjectsession_a_srcdir)/defsmanager.cpp \ + $(liblumiprocmobjectsession_a_srcdir)/fixture.cpp \ + $(liblumiprocmobjectsession_a_srcdir)/locatingpin.cpp \ + $(liblumiprocmobjectsession_a_srcdir)/mobjectfactory.cpp \ + $(liblumiprocmobjectsession_a_srcdir)/wish.cpp + + + + +noinst_HEADERS += \ + $(liblumiproc_a_srcdir)/asset/category.hpp \ + $(liblumiproc_a_srcdir)/asset/clip.hpp \ + $(liblumiproc_a_srcdir)/asset/codec.hpp \ + $(liblumiproc_a_srcdir)/asset/compoundmedia.hpp \ + $(liblumiproc_a_srcdir)/asset/dataset.hpp \ + $(liblumiproc_a_srcdir)/asset/effect.hpp \ + $(liblumiproc_a_srcdir)/asset/preview.hpp \ + $(liblumiproc_a_srcdir)/asset/db.hpp \ + $(liblumiproc_a_srcdir)/asset/buildinstruct.hpp \ + $(liblumiproc_a_srcdir)/asset/media.hpp \ + $(liblumiproc_a_srcdir)/asset/meta.hpp \ + $(liblumiproc_a_srcdir)/asset/pipe.hpp \ + $(liblumiproc_a_srcdir)/asset/proc.hpp \ + $(liblumiproc_a_srcdir)/asset/procpatt.hpp \ + $(liblumiproc_a_srcdir)/asset/struct.hpp \ + $(liblumiproc_a_srcdir)/asset/structfactoryimpl.hpp \ + $(liblumiproc_a_srcdir)/asset/track.hpp \ + $(liblumiproc_a_srcdir)/asset/unknown.hpp \ + $(liblumiproc_a_srcdir)/controllerfacade.hpp \ + $(liblumiproc_a_srcdir)/engine/aframe.hpp \ + $(liblumiproc_a_srcdir)/engine/arender.hpp \ + $(liblumiproc_a_srcdir)/engine/codecadapter.hpp \ + $(liblumiproc_a_srcdir)/engine/exitnode.hpp \ + $(liblumiproc_a_srcdir)/engine/glbuf.hpp \ + $(liblumiproc_a_srcdir)/engine/glpipe.hpp \ + $(liblumiproc_a_srcdir)/engine/glrender.hpp \ + $(liblumiproc_a_srcdir)/engine/hub.hpp \ + $(liblumiproc_a_srcdir)/engine/link.hpp \ + $(liblumiproc_a_srcdir)/engine/mask.hpp \ + $(liblumiproc_a_srcdir)/engine/pluginadapter.hpp \ + $(liblumiproc_a_srcdir)/engine/procnode.hpp \ + $(liblumiproc_a_srcdir)/engine/projector.hpp \ + $(liblumiproc_a_srcdir)/engine/renderengine.hpp \ + $(liblumiproc_a_srcdir)/engine/source.hpp \ + $(liblumiproc_a_srcdir)/engine/trafo.hpp \ + $(liblumiproc_a_srcdir)/engine/vframe.hpp \ + $(liblumiproc_a_srcdir)/engine/vrender.hpp \ + $(liblumiproc_a_srcdir)/engine/processor.hpp \ + $(liblumiproc_a_srcdir)/frame.hpp \ + $(liblumiproc_a_srcdir)/mobject/builder/assembler.hpp \ + $(liblumiproc_a_srcdir)/mobject/builder/buildertool.hpp \ + $(liblumiproc_a_srcdir)/mobject/builder/conmanager.hpp \ + $(liblumiproc_a_srcdir)/mobject/builder/nodecreatertool.hpp \ + $(liblumiproc_a_srcdir)/mobject/builder/segmentationtool.hpp \ + $(liblumiproc_a_srcdir)/mobject/builder/toolfactory.hpp \ + $(liblumiproc_a_srcdir)/mobject/builderfacade.hpp \ + $(liblumiproc_a_srcdir)/mobject/controller/pathmanager.hpp \ + $(liblumiproc_a_srcdir)/mobject/controller/renderstate.hpp \ + $(liblumiproc_a_srcdir)/mobject/interpolator.hpp \ + $(liblumiproc_a_srcdir)/mobject/parameter.hpp \ + $(liblumiproc_a_srcdir)/mobject/paramprovider.hpp \ + $(liblumiproc_a_srcdir)/mobject/session/abstractmo.hpp \ + $(liblumiproc_a_srcdir)/mobject/session/allocation.hpp \ + $(liblumiproc_a_srcdir)/mobject/session/auto.hpp \ + $(liblumiproc_a_srcdir)/mobject/session/compoundclip.hpp \ + $(liblumiproc_a_srcdir)/mobject/session/effect.hpp \ + $(liblumiproc_a_srcdir)/mobject/session/fixedlocation.hpp \ + $(liblumiproc_a_srcdir)/mobject/session/label.hpp \ + $(liblumiproc_a_srcdir)/mobject/session/meta.hpp \ + $(liblumiproc_a_srcdir)/mobject/session/relativelocation.hpp \ + $(liblumiproc_a_srcdir)/mobject/session/simpleclip.hpp \ + $(liblumiproc_a_srcdir)/mobject/session/mobjectfactory.hpp \ + $(liblumiproc_a_srcdir)/mobject/session/segment.hpp \ + $(liblumiproc_a_srcdir)/mobject/session/track.hpp \ + $(liblumiproc_a_srcdir)/mobject/session/wish.hpp \ + $(liblumiproc_a_srcdir)/mobject/session/clip.hpp \ + $(liblumiproc_a_srcdir)/mobject/session/constraint.hpp \ + $(liblumiproc_a_srcdir)/mobject/session/defsmanager.hpp \ + $(liblumiproc_a_srcdir)/mobject/session/edl.hpp \ + $(liblumiproc_a_srcdir)/mobject/session/fixture.hpp \ + $(liblumiproc_a_srcdir)/mobject/session/locatingpin.hpp \ + $(liblumiproc_a_srcdir)/mobject/session/plug.hpp \ + $(liblumiproc_a_srcdir)/mobject/session/sessionimpl.hpp \ + $(liblumiproc_a_srcdir)/mobject/session/defsregistry.hpp \ + $(liblumiproc_a_srcdir)/mobject/explicitplacement.hpp \ + $(liblumiproc_a_srcdir)/mobject/mobject.hpp \ + $(liblumiproc_a_srcdir)/mobject/placement.hpp \ + $(liblumiproc_a_srcdir)/mobject/session.hpp \ + $(liblumiproc_a_srcdir)/stateproxy.hpp \ + $(liblumiproc_a_srcdir)/asset.hpp \ + $(liblumiproc_a_srcdir)/assetmanager.hpp \ + $(liblumiproc_a_srcdir)/lumiera.hpp \ + $(liblumiproc_a_srcdir)/nobugcfg.hpp + From b6bd181ca8bd05d736ed215f2945562c3a39dd14 Mon Sep 17 00:00:00 2001 From: Christian Thaeter Date: Mon, 19 May 2008 01:48:02 +0200 Subject: [PATCH 06/39] correct pasting glitch and copyright header date --- src/proc/Makefile.am | 12 ------------ tests/Makefile.am | 2 +- 2 files changed, 1 insertion(+), 13 deletions(-) diff --git a/src/proc/Makefile.am b/src/proc/Makefile.am index 612ffc2d6..603f4d825 100644 --- a/src/proc/Makefile.am +++ b/src/proc/Makefile.am @@ -10,18 +10,6 @@ # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. -# Copyright (C) Lumiera.org -# 2008, Christian Thaeter -# -# This program is free software; you can redistribute it and/or -# modify it under the terms of the GNU General Public License as -# published by the Free Software Foundation; either version 2 of the -# License, or (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program; if not, write to the Free Software diff --git a/tests/Makefile.am b/tests/Makefile.am index a6ccae010..1dce33db8 100644 --- a/tests/Makefile.am +++ b/tests/Makefile.am @@ -1,5 +1,5 @@ # Copyright (C) Lumiera.org -# 2007, Christian Thaeter +# 2007-2008, Christian Thaeter # Hermann Vosseler # # This program is free software; you can redistribute it and/or From 3b6713b23c2612f8ecfe19ff647e8395ced23ee5 Mon Sep 17 00:00:00 2001 From: Christian Thaeter Date: Mon, 19 May 2008 01:50:53 +0200 Subject: [PATCH 07/39] build common and (re-)enable backend build (C++ part) - not (yet) merged backend C parts are commented out for now --- Makefile.am | 3 +- src/backend/Makefile.am | 25 +++++++++------- src/common/Makefile.am | 65 +++++++++++++++++++++++++++++++++++++++++ 3 files changed, 82 insertions(+), 11 deletions(-) create mode 100644 src/common/Makefile.am diff --git a/Makefile.am b/Makefile.am index eb67e0a23..2ea3aba54 100644 --- a/Makefile.am +++ b/Makefile.am @@ -34,8 +34,9 @@ include $(top_srcdir)/admin/Makefile.am # core include $(top_srcdir)/src/proc/Makefile.am +include $(top_srcdir)/src/common/Makefile.am include $(top_srcdir)/src/lib/Makefile.am -#include $(top_srcdir)/src/backend/Makefile.am +include $(top_srcdir)/src/backend/Makefile.am # plugins #include $(top_srcdir)/src... diff --git a/src/backend/Makefile.am b/src/backend/Makefile.am index 5f693f8bf..3d70a09d5 100644 --- a/src/backend/Makefile.am +++ b/src/backend/Makefile.am @@ -22,16 +22,21 @@ liblumibackend_a_CFLAGS = $(CFLAGS) -std=gnu99 -Wall -Werror liblumibackend_a_CPPFLAGS = -I$(top_srcdir)/src/ liblumibackend_a_SOURCES = \ - $(liblumibackend_a_srcdir)/backend.c \ - $(liblumibackend_a_srcdir)/file.c \ - $(liblumibackend_a_srcdir)/filehandle.c \ - $(liblumibackend_a_srcdir)/filedescriptor.c \ - $(liblumibackend_a_srcdir)/filehandlecache.c + $(liblumibackend_a_srcdir)/mediaaccessfacade.cpp + +# $(liblumibackend_a_srcdir)/backend.c \ +# $(liblumibackend_a_srcdir)/file.c \ +# $(liblumibackend_a_srcdir)/filehandle.c \ +# $(liblumibackend_a_srcdir)/filedescriptor.c \ +# $(liblumibackend_a_srcdir)/filehandlecache.c + noinst_HEADERS += \ - $(liblumibackend_a_srcdir)/backend.h \ - $(liblumibackend_a_srcdir)/file.h \ - $(liblumibackend_a_srcdir)/filehandle.h \ - $(liblumibackend_a_srcdir)/filedescriptor.h \ - $(liblumibackend_a_srcdir)/filehandlecache.h + $(liblumibackend_a_srcdir)mediaaccessfacade.cpp + +# $(liblumibackend_a_srcdir)/backend.h \ +# $(liblumibackend_a_srcdir)/file.h \ +# $(liblumibackend_a_srcdir)/filehandle.h \ +# $(liblumibackend_a_srcdir)/filedescriptor.h \ +# $(liblumibackend_a_srcdir)/filehandlecache.h diff --git a/src/common/Makefile.am b/src/common/Makefile.am new file mode 100644 index 000000000..06082da31 --- /dev/null +++ b/src/common/Makefile.am @@ -0,0 +1,65 @@ +# Copyright (C) Lumiera.org +# 2008, Christian Thaeter +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License as +# published by the Free Software Foundation; either version 2 of the +# License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA. + + +# TODO use -Wextra -Werror + +liblumicommon_a_srcdir = $(top_srcdir)/src/common +noinst_LIBRARIES += liblumicommon.a + +liblumicommon_a_CXXFLAGS = $(CXXFLAGS) -Wall +liblumicommon_a_CPPFLAGS = -I$(top_srcdir)/src/ + +liblumicommon_a_SOURCES = \ + $(liblumicommon_a_srcdir)/time.cpp \ + $(liblumicommon_a_srcdir)/util.cpp \ + $(liblumicommon_a_srcdir)/visitor.cpp \ + $(liblumicommon_a_srcdir)/cmdline.cpp \ + $(liblumicommon_a_srcdir)/configrules.cpp \ + $(liblumicommon_a_srcdir)/error.cpp \ + $(liblumicommon_a_srcdir)/query.cpp \ + $(liblumicommon_a_srcdir)/query/mockconfigrules.cpp \ + $(liblumicommon_a_srcdir)/test/suite.cpp \ + $(liblumicommon_a_srcdir)/test/testoption.cpp + + +noinst_HEADERS += \ + $(liblumicommon_a_srcdir)/cmdline.hpp \ + $(liblumicommon_a_srcdir)/factory.hpp \ + $(liblumicommon_a_srcdir)/singleton.hpp \ + $(liblumicommon_a_srcdir)/singletonpolicies.hpp \ + $(liblumicommon_a_srcdir)/singletonpreconfigure.hpp \ + $(liblumicommon_a_srcdir)/time.hpp \ + $(liblumicommon_a_srcdir)/typelist.hpp \ + $(liblumicommon_a_srcdir)/visitor.hpp \ + $(liblumicommon_a_srcdir)/visitordispatcher.hpp \ + $(liblumicommon_a_srcdir)/visitorpolicies.hpp \ + $(liblumicommon_a_srcdir)/configrules.hpp \ + $(liblumicommon_a_srcdir)/error.hpp \ + $(liblumicommon_a_srcdir)/multithread.hpp \ + $(liblumicommon_a_srcdir)/p.hpp \ + $(liblumicommon_a_srcdir)/query.hpp \ + $(liblumicommon_a_srcdir)/query/mockconfigrules.hpp \ + $(liblumicommon_a_srcdir)/singletonfactory.hpp \ + $(liblumicommon_a_srcdir)/singletonsubclass.hpp \ + $(liblumicommon_a_srcdir)/typelistutil.hpp \ + $(liblumicommon_a_srcdir)/util.hpp \ + $(liblumicommon_a_srcdir)/test/mockinjector.hpp \ + $(liblumicommon_a_srcdir)/test/suite.hpp \ + $(liblumicommon_a_srcdir)/test/testoption.hpp \ + $(liblumicommon_a_srcdir)/test/run.hpp + From 5ae909e2a0f7b9e445a95329d7cc15b09cfef912 Mon Sep 17 00:00:00 2001 From: Christian Thaeter Date: Mon, 19 May 2008 01:51:34 +0200 Subject: [PATCH 08/39] add C++ parts to lib build --- src/lib/Makefile.am | 50 +++++++++++++++++++++++---------------------- 1 file changed, 26 insertions(+), 24 deletions(-) diff --git a/src/lib/Makefile.am b/src/lib/Makefile.am index 0e0c2f22e..ffdb9a176 100644 --- a/src/lib/Makefile.am +++ b/src/lib/Makefile.am @@ -21,28 +21,30 @@ noinst_LIBRARIES += liblumi.a liblumi_a_CFLAGS = $(CFLAGS) -std=gnu99 -Wall -Werror liblumi_a_CPPFLAGS = -I$(top_srcdir)/src/ -liblumi_a_SOURCES = \ - $(liblumi_a_srcdir)/plugin.c \ - $(liblumi_a_srcdir)/error.c \ - $(liblumi_a_srcdir)/mutex.c \ - $(liblumi_a_srcdir)/rwlock.c \ - $(liblumi_a_srcdir)/condition.c \ - $(liblumi_a_srcdir)/references.c \ - $(liblumi_a_srcdir)/uuid.c \ - $(liblumi_a_srcdir)/safeclib.c \ - $(liblumi_a_srcdir)/cuckoo.c \ - $(liblumi_a_srcdir)/mrucache.c +liblumi_a_SOURCES = \ + $(liblumi_a_srcdir)/plugin.c \ + $(liblumi_a_srcdir)/error.c \ + $(liblumi_a_srcdir)/mutex.c \ + $(liblumi_a_srcdir)/rwlock.c \ + $(liblumi_a_srcdir)/condition.c \ + $(liblumi_a_srcdir)/references.c \ + $(liblumi_a_srcdir)/uuid.c \ + $(liblumi_a_srcdir)/safeclib.c \ + $(liblumi_a_srcdir)/cuckoo.c \ + $(liblumi_a_srcdir)/mrucache.c \ + $(liblumi_a_srcdir)/appconfig.cpp - -noinst_HEADERS += \ - $(liblumi_a_srcdir)/plugin.h \ - $(liblumi_a_srcdir)/error.h \ - $(liblumi_a_srcdir)/locking.h \ - $(liblumi_a_srcdir)/mutex.h \ - $(liblumi_a_srcdir)/rwlock.h \ - $(liblumi_a_srcdir)/condition.h \ - $(liblumi_a_srcdir)/references.h \ - $(liblumi_a_srcdir)/uuid.h \ - $(liblumi_a_srcdir)/safeclib.h \ - $(liblumi_a_srcdir)/cuckoo.h \ - $(liblumi_a_srcdir)/mrucache.h +noinst_HEADERS += \ + $(liblumi_a_srcdir)/plugin.h \ + $(liblumi_a_srcdir)/error.h \ + $(liblumi_a_srcdir)/locking.h \ + $(liblumi_a_srcdir)/mutex.h \ + $(liblumi_a_srcdir)/rwlock.h \ + $(liblumi_a_srcdir)/condition.h \ + $(liblumi_a_srcdir)/references.h \ + $(liblumi_a_srcdir)/uuid.h \ + $(liblumi_a_srcdir)/safeclib.h \ + $(liblumi_a_srcdir)/cuckoo.h \ + $(liblumi_a_srcdir)/mrucache.h \ + $(liblumi_a_srcdir)/appconfig.hpp \ + $(liblumi_a_srcdir)/lifecycleregistry.hpp From 8ba3df3b31f34580e199bb306f0726a21e22f69c Mon Sep 17 00:00:00 2001 From: Christian Thaeter Date: Mon, 19 May 2008 01:54:40 +0200 Subject: [PATCH 09/39] missing slash --- src/backend/Makefile.am | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/backend/Makefile.am b/src/backend/Makefile.am index 3d70a09d5..eb5853d5f 100644 --- a/src/backend/Makefile.am +++ b/src/backend/Makefile.am @@ -32,7 +32,7 @@ liblumibackend_a_SOURCES = \ noinst_HEADERS += \ - $(liblumibackend_a_srcdir)mediaaccessfacade.cpp + $(liblumibackend_a_srcdir)/mediaaccessfacade.cpp # $(liblumibackend_a_srcdir)/backend.h \ # $(liblumibackend_a_srcdir)/file.h \ From d6d6a1040499e59274fe027fdf5c536a3562395f Mon Sep 17 00:00:00 2001 From: Christian Thaeter Date: Mon, 19 May 2008 01:55:05 +0200 Subject: [PATCH 10/39] add components test build --- Makefile.am | 1 + tests/components/Makefile.am | 72 ++++++++++++++++++++++++++++++++++++ 2 files changed, 73 insertions(+) create mode 100644 tests/components/Makefile.am diff --git a/Makefile.am b/Makefile.am index 2ea3aba54..79d054de2 100644 --- a/Makefile.am +++ b/Makefile.am @@ -42,6 +42,7 @@ include $(top_srcdir)/src/backend/Makefile.am #include $(top_srcdir)/src... # tests +include $(top_srcdir)/tests/components/Makefile.am include $(top_srcdir)/tests/Makefile.am include $(top_srcdir)/tests/plugin/Makefile.am diff --git a/tests/components/Makefile.am b/tests/components/Makefile.am new file mode 100644 index 000000000..5e8d1e070 --- /dev/null +++ b/tests/components/Makefile.am @@ -0,0 +1,72 @@ +# Copyright (C) Lumiera.org +# 2007, Christian Thaeter +# Hermann Vosseler +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License as +# published by the Free Software Foundation; either version 2 of the +# License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA. + +testcomponents_srcdir = $(top_srcdir)/tests/components + +check_PROGRAMS += test-components +test_components_CPPFLAGS = $(AM_CPPFLAGS) -Wall -I$(top_srcdir)/src -I$(testcomponents_srcdir) +test_components_LDADD = \ + liblumiproc.a \ + liblumiprocengine.a \ + liblumiprocmobjectbuilder.a \ + liblumiprocmobjectsession.a \ + liblumiprocasset.a \ + liblumiprocmobject.a \ + liblumiprocmobjectcontroller.a \ + liblumi.a \ + liblumicommon.a \ + liblumibackend.a \ + -lnobugmt -lpthread -ldl -lboost_program_options-mt -lboost_regex-mt + +test_components_SOURCES = \ + $(testcomponents_srcdir)/backend/mediaaccessmock.cpp \ + $(testcomponents_srcdir)/backend/mediaaccessmocktest.cpp \ + $(testcomponents_srcdir)/proc/asset/assetcategorytest.cpp \ + $(testcomponents_srcdir)/proc/asset/compoundmediatest.cpp \ + $(testcomponents_srcdir)/proc/asset/deleteassettest.cpp \ + $(testcomponents_srcdir)/proc/asset/identityofassetstest.cpp \ + $(testcomponents_srcdir)/proc/asset/mediastructurequerytest.cpp \ + $(testcomponents_srcdir)/proc/asset/basicpipetest.cpp \ + $(testcomponents_srcdir)/proc/asset/createassettest.cpp \ + $(testcomponents_srcdir)/proc/asset/dependantassetstest.cpp \ + $(testcomponents_srcdir)/proc/asset/makecliptest.cpp \ + $(testcomponents_srcdir)/proc/asset/orderingofassetstest.cpp \ + $(testcomponents_srcdir)/proc/asset/testasset.cpp \ + $(testcomponents_srcdir)/proc/engine/sourcenodetest.cpp \ + $(testcomponents_srcdir)/proc/mobject/builder/buildertooltest.cpp \ + $(testcomponents_srcdir)/proc/mobject/builder/buildsegmenttest.cpp \ + $(testcomponents_srcdir)/proc/mobject/controller/rendersegmenttest.cpp \ + $(testcomponents_srcdir)/proc/mobject/placementbasictest.cpp \ + $(testcomponents_srcdir)/proc/mobject/session/addcliptest.cpp \ + $(testcomponents_srcdir)/proc/mobject/session/deletecliptest.cpp \ + $(testcomponents_srcdir)/proc/mobject/session/rebuildfixturetest.cpp \ + $(testcomponents_srcdir)/proc/mobject/session/sessionmanagertest.cpp \ + $(testcomponents_srcdir)/proc/mobject/session/testclip.cpp \ + $(testcomponents_srcdir)/proc/mobject/session/defsmanagerimpltest.cpp \ + $(testcomponents_srcdir)/proc/mobject/session/defsmanagertest.cpp \ + $(testcomponents_srcdir)/proc/mobject/session/defsregistryimpltest.cpp \ + $(testcomponents_srcdir)/proc/mobject/session/sessionstructuretest.cpp \ + $(testcomponents_srcdir)/mainsuite.cpp + +noinst_HEADERS += \ + $(testcomponents_srcdir)/backend/mediaaccessmock.hpp \ + $(testcomponents_srcdir)/proc/asset/testclipasset.hpp \ + $(testcomponents_srcdir)/proc/asset/assetdiagnostics.hpp \ + $(testcomponents_srcdir)/proc/asset/testasset.hpp \ + $(testcomponents_srcdir)/proc/mobject/session/testclip.hpp \ + $(testcomponents_srcdir)/proc/mobject/session/testsession1.hpp From e1b8849d0c23896d5bd18fceeb1fda1c0057e643 Mon Sep 17 00:00:00 2001 From: Christian Thaeter Date: Mon, 19 May 2008 13:30:20 +0200 Subject: [PATCH 11/39] build 'common' tests --- Makefile.am | 1 + tests/common/Makefile.am | 61 ++++++++++++++++++++++++++++++++++++++++ 2 files changed, 62 insertions(+) create mode 100644 tests/common/Makefile.am diff --git a/Makefile.am b/Makefile.am index 79d054de2..e6f29bf97 100644 --- a/Makefile.am +++ b/Makefile.am @@ -42,6 +42,7 @@ include $(top_srcdir)/src/backend/Makefile.am #include $(top_srcdir)/src... # tests +include $(top_srcdir)/tests/common/Makefile.am include $(top_srcdir)/tests/components/Makefile.am include $(top_srcdir)/tests/Makefile.am include $(top_srcdir)/tests/plugin/Makefile.am diff --git a/tests/common/Makefile.am b/tests/common/Makefile.am new file mode 100644 index 000000000..cc027e477 --- /dev/null +++ b/tests/common/Makefile.am @@ -0,0 +1,61 @@ +# Copyright (C) Lumiera.org +# 2008, Christian Thaeter +# Hermann Vosseler +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License as +# published by the Free Software Foundation; either version 2 of the +# License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA. + +testcommon_srcdir = $(top_srcdir)/tests/common + +check_PROGRAMS += test-common +test_common_CPPFLAGS = $(AM_CPPFLAGS) -Wall -I$(top_srcdir)/src -I$(testcommon_srcdir) +test_common_LDADD = \ + liblumiproc.a \ + liblumiprocengine.a \ + liblumiprocmobjectbuilder.a \ + liblumiprocmobjectsession.a \ + liblumiprocasset.a \ + liblumiprocmobject.a \ + liblumiprocmobjectcontroller.a \ + liblumi.a \ + liblumicommon.a \ + liblumibackend.a \ + -lnobugmt -lpthread -ldl -lboost_program_options-mt -lboost_regex-mt + +test_common_SOURCES = \ + $(testcommon_srcdir)/appconfigtest.cpp \ + $(testcommon_srcdir)/customsharedptrtest.cpp \ + $(testcommon_srcdir)/exceptionerrortest.cpp \ + $(testcommon_srcdir)/factoryspecialtest.cpp \ + $(testcommon_srcdir)/factorytest.cpp \ + $(testcommon_srcdir)/helloworldtest.cpp \ + $(testcommon_srcdir)/lifecycletest.cpp \ + $(testcommon_srcdir)/mainsuite.cpp \ + $(testcommon_srcdir)/query/queryutilstest.cpp \ + $(testcommon_srcdir)/removefromsettest.cpp \ + $(testcommon_srcdir)/sanitizedidentifiertest.cpp \ + $(testcommon_srcdir)/singletonsubclasstest.cpp \ + $(testcommon_srcdir)/singletontest.cpp \ + $(testcommon_srcdir)/singletontestmocktest.cpp \ + $(testcommon_srcdir)/test/cmdlinewrappertest.cpp \ + $(testcommon_srcdir)/test/testoptiontest.cpp \ + $(testcommon_srcdir)/typelisttest.cpp \ + $(testcommon_srcdir)/typelistutiltest.cpp \ + $(testcommon_srcdir)/visitingtoolconcept.cpp \ + $(testcommon_srcdir)/visitingtoolextendedtest.cpp \ + $(testcommon_srcdir)/visitingtooltest.cpp + +noinst_HEADERS += \ + $(testcommon_srcdir)/query/querydiagnostics.hpp \ + $(testcommon_srcdir)/testtargetobj.hpp From bc436d1d6a0cf75c66050f317020b366dba69b44 Mon Sep 17 00:00:00 2001 From: Christian Thaeter Date: Mon, 19 May 2008 13:30:55 +0200 Subject: [PATCH 12/39] fix typo in component test --- tests/50components.tests | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/50components.tests b/tests/50components.tests index 2bac409cb..18f04ee4b 100644 --- a/tests/50components.tests +++ b/tests/50components.tests @@ -62,7 +62,7 @@ out: caught error::Logic: LUMIERA_ERROR_FATAL:floundered (test-2). out: caught error::Invalid: LUMIERA_ERROR_INVALID:invalid input or parameters (test-3). out: caught lumiera::Error: LUMIERA_ERROR_EXTERNAL:failure in external service (test-4). out: caught std::runtime_error: test-5 -out: caught std::exception: St9exception +out: caught std::exception: std::exception out: intermediate handler caught: LUMIERA_ERROR_EXTERNAL:failure in external service (test-7).....will rethrow as error::State out: caught lumiera::Error: LUMIERA_ERROR_STATE:unforseen state -- caused by: LUMIERA_ERROR_EXTERNAL:failure in external service (test-7). out: intermediate handler caught: LUMIERA_ERROR_EXTERNAL:failure in external service (test-8).....will rethrow as error::State From 5659219e55c3348b7ea1aab8e7c3b823c172aaf4 Mon Sep 17 00:00:00 2001 From: Christian Thaeter Date: Mon, 19 May 2008 13:31:18 +0200 Subject: [PATCH 13/39] correct date in copyright header --- tests/components/Makefile.am | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/components/Makefile.am b/tests/components/Makefile.am index 5e8d1e070..088dbe668 100644 --- a/tests/components/Makefile.am +++ b/tests/components/Makefile.am @@ -1,5 +1,5 @@ # Copyright (C) Lumiera.org -# 2007, Christian Thaeter +# 2008, Christian Thaeter # Hermann Vosseler # # This program is free software; you can redistribute it and/or From b4183ed44456c830e61a848e09b3a34fd6c70b9f Mon Sep 17 00:00:00 2001 From: Christian Thaeter Date: Tue, 1 Apr 2008 07:55:20 +0200 Subject: [PATCH 14/39] safeclib improvements * add lumiera_calloc() * improve cleanup functions for tmpbuf --- src/lib/safeclib.c | 40 ++++++++++++++++++++++++++-------------- src/lib/safeclib.h | 1 - 2 files changed, 26 insertions(+), 15 deletions(-) diff --git a/src/lib/safeclib.c b/src/lib/safeclib.c index a0b562643..934c3ff5b 100644 --- a/src/lib/safeclib.c +++ b/src/lib/safeclib.c @@ -1,7 +1,7 @@ /* safe_clib.c - Portable and safe wrapers around some clib functions and some tools - Copyright (C) CinelerraCV + Copyright (C) Lumiera.org 2008, Christian Thaeter This program is free software; you can redistribute it and/or @@ -39,9 +39,26 @@ LUMIERA_ERROR_DEFINE (NO_MEMORY, "Out of Memory!"); * @return pointer to the allocated memory */ void* -lumiera_malloc (size_t sz) +lumiera_malloc (size_t size) { - void* o = sz? malloc (sz) : NULL; + void* o = size ? malloc (size) : NULL; + if (!o) + LUMIERA_DIE (NO_MEMORY); + return o; +} + + +/** + * Allocate cleared memory for an array. + * always succeeds or dies + * @param n number of elements + * @param size memory to be allocated + * @return pointer to the allocated memory + */ +void* +lumiera_calloc (size_t n, size_t size) +{ + void* o = (n&&size)? calloc (n, size) : NULL; if (!o) LUMIERA_DIE (NO_MEMORY); return o; @@ -120,7 +137,6 @@ static void lumiera_tmpbuf_destroy (void* buf) { lumiera_tmpbuf_freeall (); - free (buf); } static void @@ -139,17 +155,13 @@ lumiera_tmpbuf_freeall (void) { pthread_once (&lumiera_tmpbuf_tls_once, lumiera_tmpbuf_init); struct lumiera_tmpbuf_struct* buf = pthread_getspecific (lumiera_tmpbuf_tls_key); - if (!buf) - pthread_setspecific (lumiera_tmpbuf_tls_key, - buf = lumiera_malloc (sizeof (struct lumiera_tmpbuf_struct))); - - for (buf->idx = 0; buf->idx < 64; ++buf->idx) + if (buf) { - free (buf->buffers[buf->idx]); - buf->buffers[buf->idx] = NULL; - buf->sizes[buf->idx] = 0; + pthread_setspecific (lumiera_tmpbuf_tls_key, NULL); + for (int idx = 0; idx < 64; ++idx) + free (buf->buffers[idx]); + free (buf); } - buf->idx = 0; } @@ -165,7 +177,7 @@ lumiera_tmpbuf_provide (size_t size) struct lumiera_tmpbuf_struct* buf = pthread_getspecific (lumiera_tmpbuf_tls_key); if (!buf) pthread_setspecific (lumiera_tmpbuf_tls_key, - buf = lumiera_malloc (sizeof (struct lumiera_tmpbuf_struct))); + buf = lumiera_calloc (1, sizeof (struct lumiera_tmpbuf_struct))); buf->idx = (buf->idx + 1) & 0x3f; diff --git a/src/lib/safeclib.h b/src/lib/safeclib.h index 02d931cb7..1aa806fd1 100644 --- a/src/lib/safeclib.h +++ b/src/lib/safeclib.h @@ -84,7 +84,6 @@ lumiera_streq (const char* a, const char* b); void lumiera_tmpbuf_freeall (void); - /** * Query a thread local tmpbuf. * @param size minimal needed size for the tmpbuf From 293b872b5d89e7a32dbfbd8b998c7476b5cd3a3e Mon Sep 17 00:00:00 2001 From: Christian Thaeter Date: Fri, 4 Apr 2008 07:58:11 +0200 Subject: [PATCH 15/39] extend the MUTEX_SECTION macro with NoBug resource tracking LUMIERA_MUTEX_SECTION takes now a nobug flag as first argument and a nobug resource-handle as second argument. This change works forward for the NoBug resource tracker and deadlock detector. --- src/lib/mutex.h | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/src/lib/mutex.h b/src/lib/mutex.h index 9381c3ca1..ca48ea592 100644 --- a/src/lib/mutex.h +++ b/src/lib/mutex.h @@ -27,12 +27,15 @@ /** * @file Mutual exclusion locking, header. */ - -#define LUMIERA_MUTEX_SECTION(mutex) \ +#define LUMIERA_MUTEX_SECTION(flag, handle, mutex) \ +RESOURCE_HANDLE (rh_##__LINE__##_); \ lumiera_mutexacquirer lock_##__LINE__##_; \ +RESOURCE_ENTER (flag, handle, "acquire mutex", &lock_##__LINE__##_, \ + NOBUG_RESOURCE_EXCLUSIVE, rh_##__LINE__##_); \ for (lumiera_mutexacquirer_init_mutex (&lock_##__LINE__##_, mutex, LUMIERA_LOCKED); \ lock_##__LINE__##_.state == LUMIERA_LOCKED; \ - lumiera_mutexacquirer_unlock (&lock_##__LINE__##_)) + lumiera_mutexacquirer_unlock (&lock_##__LINE__##_), \ + ({RESOURCE_LEAVE(flag, rh_##__LINE__##_);})) /** From b909c2247d3ef8c545c96e534db6d025038ec445 Mon Sep 17 00:00:00 2001 From: Christian Thaeter Date: Mon, 7 Apr 2008 05:43:40 +0200 Subject: [PATCH 16/39] add a cuckoo_nelements function Query the number of elements stored in a hash, useful for debugging. --- src/lib/cuckoo.c | 7 +++++++ src/lib/cuckoo.h | 7 +++++++ 2 files changed, 14 insertions(+) diff --git a/src/lib/cuckoo.c b/src/lib/cuckoo.c index 0d0b4efef..bbcb0e71a 100644 --- a/src/lib/cuckoo.c +++ b/src/lib/cuckoo.c @@ -208,6 +208,13 @@ cuckoo_insert_internal_ (Cuckoo self, void* item) } +size_t +cuckoo_nelements (Cuckoo self) +{ + return self->elements; +} + + static void cuckoo_rehash (Cuckoo self) { diff --git a/src/lib/cuckoo.h b/src/lib/cuckoo.h index db1d33da3..63b6d260d 100644 --- a/src/lib/cuckoo.h +++ b/src/lib/cuckoo.h @@ -105,6 +105,13 @@ cuckoo_destroy (Cuckoo self); void cuckoo_free (Cuckoo self); +/** + * Get the number of elements stored in a hash. + * @return number of elements, 0 when empty + */ +size_t +cuckoo_nelements (Cuckoo self); + /** * Insert an element into a hash. * amortized O(1) complexity because it may rarely rehash the tables or even grow them. From 0996c386bbab0250fda9fcde0061ecc8c8f26156 Mon Sep 17 00:00:00 2001 From: Christian Thaeter Date: Mon, 7 Apr 2008 06:38:05 +0200 Subject: [PATCH 17/39] Mrucache update * Fix: unlinking at destroy * rename add/remove to checkin/checkout * add a drop function which moves a node to the end for fast reuse --- src/lib/mrucache.c | 1 + src/lib/mrucache.h | 30 +++++++++++++++++++++++++++--- 2 files changed, 28 insertions(+), 3 deletions(-) diff --git a/src/lib/mrucache.c b/src/lib/mrucache.c index d98976184..b05169db8 100644 --- a/src/lib/mrucache.c +++ b/src/lib/mrucache.c @@ -43,6 +43,7 @@ lumiera_mrucache_destroy (LumieraMruCache self) { LLIST_WHILE_TAIL (&self->cache_list, node) { + llist_unlink (node); if (self->destructor_cb) free (self->destructor_cb (node)); else diff --git a/src/lib/mrucache.h b/src/lib/mrucache.h index 8af7434d9..4644abe8d 100644 --- a/src/lib/mrucache.h +++ b/src/lib/mrucache.h @@ -67,7 +67,7 @@ lumiera_mrucache_destroy (LumieraMruCache self); * add an element to a mrucache */ static inline void -lumiera_mrucache_add (LumieraMruCache self, LList node) +lumiera_mrucache_checkin (LumieraMruCache self, LList node) { REQUIRE (self); REQUIRE (node && llist_is_empty (node)); @@ -76,11 +76,34 @@ lumiera_mrucache_add (LumieraMruCache self, LList node) } +/** + * place an element at the tail of a mrucache, to be reused as first + */ +static inline void +lumiera_mrucache_drop (LumieraMruCache self, LList node) +{ + REQUIRE (self); + REQUIRE (node); + + if (llist_is_empty (node)) + { + /* was not in list, we need to count it */ + ++self->cached; + } + else + { + /* speedup loop warning :P, this check is costly */ + REQUIRE (llist_is_member (&self->cache_list, node), "node must be empty or member of cache"); + } + llist_insert_tail (&self->cache_list, node); +} + + /** * Remove an element from a mrucache. */ static inline void -lumiera_mrucache_remove (LumieraMruCache self, LList node) +lumiera_mrucache_checkout (LumieraMruCache self, LList node) { REQUIRE (self); REQUIRE (node && llist_is_member (&self->cache_list, node)); /* speedup loop warning :P, this check is costly */ @@ -88,6 +111,7 @@ lumiera_mrucache_remove (LumieraMruCache self, LList node) --self->cached; } + /** * destroy the oldest element and return a pointer to it for reuse. */ @@ -98,7 +122,7 @@ lumiera_mrucache_pop (LumieraMruCache self) if (llist_is_empty (&self->cache_list)) return NULL; - LList node = llist_tail (&self->cache_list); + LList node = llist_tail (&self->cache_list); llist_unlink (node); --self->cached; From ff6ce603a4b683210449efc92dc14257f70fdfb4 Mon Sep 17 00:00:00 2001 From: Christian Thaeter Date: Mon, 7 Apr 2008 19:21:52 +0200 Subject: [PATCH 18/39] tmpbuf_sprintf renamed to tmpbuf_snprintf --- src/lib/safeclib.c | 2 +- src/lib/safeclib.h | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/lib/safeclib.c b/src/lib/safeclib.c index 934c3ff5b..1240cb673 100644 --- a/src/lib/safeclib.c +++ b/src/lib/safeclib.c @@ -206,7 +206,7 @@ lumiera_tmpbuf_strndup (const char* src, size_t size) char* -lumiera_tmpbuf_sprintf (size_t size, const char* fmt, ...) +lumiera_tmpbuf_snprintf (size_t size, const char* fmt, ...) { va_list args; diff --git a/src/lib/safeclib.h b/src/lib/safeclib.h index 1aa806fd1..2c835c8c2 100644 --- a/src/lib/safeclib.h +++ b/src/lib/safeclib.h @@ -109,5 +109,5 @@ lumiera_tmpbuf_strndup (const char* src, size_t size); * @return temporary buffer containing the constructed of the string */ char* -lumiera_tmpbuf_sprintf (size_t size, const char* fmt, ...); +lumiera_tmpbuf_snprintf (size_t size, const char* fmt, ...); From dc0cbf0ad8ef18b329ef4ae163b07712a363e762 Mon Sep 17 00:00:00 2001 From: Christian Thaeter Date: Mon, 7 Apr 2008 20:36:24 +0200 Subject: [PATCH 19/39] Fix: off by one allocation error in tmpbuf_snprintf() --- src/lib/safeclib.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/lib/safeclib.c b/src/lib/safeclib.c index 1240cb673..0e82e2dc0 100644 --- a/src/lib/safeclib.c +++ b/src/lib/safeclib.c @@ -215,9 +215,9 @@ lumiera_tmpbuf_snprintf (size_t size, const char* fmt, ...) va_end (args); len = len > size ? size : len; - char* buf = lumiera_tmpbuf_provide (len); + char* buf = lumiera_tmpbuf_provide (len+1); va_start (args, fmt); - vsnprintf (buf, len, fmt, args); + vsnprintf (buf, len+1, fmt, args); va_end (args); return buf; From 63c07d12749dafa3dac7a18a3d8580a949729612 Mon Sep 17 00:00:00 2001 From: Christian Thaeter Date: Wed, 9 Apr 2008 07:57:03 +0200 Subject: [PATCH 20/39] doxydocing mrucache --- src/lib/mrucache.c | 10 ---------- src/lib/mrucache.h | 50 ++++++++++++++++++++++++++++++++++++---------- 2 files changed, 40 insertions(+), 20 deletions(-) diff --git a/src/lib/mrucache.c b/src/lib/mrucache.c index b05169db8..0f9bb86df 100644 --- a/src/lib/mrucache.c +++ b/src/lib/mrucache.c @@ -21,12 +21,6 @@ #include "lib/mrucache.h" -/** - * @file Most recent used cache - * Elements (addressed by a LList node) are either in the cache and thereby subject of aging - * or checked out under control of the user. - */ - LumieraMruCache lumiera_mrucache_init (LumieraMruCache self, lumiera_cache_destructor_fn destructor_cb) @@ -53,10 +47,6 @@ lumiera_mrucache_destroy (LumieraMruCache self) return self; } - -/** - * destroy and free the nelem oldest elements - */ int lumiera_mrucache_age (LumieraMruCache self, int nelem) { diff --git a/src/lib/mrucache.h b/src/lib/mrucache.h index 4644abe8d..3edb680f0 100644 --- a/src/lib/mrucache.h +++ b/src/lib/mrucache.h @@ -28,8 +28,9 @@ /** * @file Most recent used cache - * Elements (addressed by a LList node) are either in the cache and thereby subject of aging - * or checked out under control of the user. + * Elements (addressed by a LList node) are either checked in the cache and thereby subject of aging + * or checked out under control of the user. Most operations require that the chache is locked. This locking + * must be done from elsewhere. */ /** @@ -50,21 +51,31 @@ typedef struct lumiera_mrucache_struct lumiera_mrucache; typedef lumiera_mrucache* LumieraMruCache; /** - * Initialize a mrucache + * Initialize a cache. + * @param self cache to be initialized + * @param destructor_cb function for destructing a cache node, preparing for reuse. + * @return self */ LumieraMruCache lumiera_mrucache_init (LumieraMruCache self, lumiera_cache_destructor_fn destructor_cb); - /** - * Destroy a mrucache + * Destroy a cache. + * frees all checked in items. + * @param self cache to be destroyed + * @return self */ LumieraMruCache lumiera_mrucache_destroy (LumieraMruCache self); /** - * add an element to a mrucache + * Add an element to a mrucache. + * When added the element is subject of aging and must not be accessed anymore. + * To acces elements they have to be checked out again. + * Checkin and checkout operations must be protected by a lock over the cache. + * @param self cache where to checkin + * @param node a llist member in the cached data which is used as queue in the cache */ static inline void lumiera_mrucache_checkin (LumieraMruCache self, LList node) @@ -77,7 +88,13 @@ lumiera_mrucache_checkin (LumieraMruCache self, LList node) /** - * place an element at the tail of a mrucache, to be reused as first + * Schedule an element for fast aging. + * When an element is not longer needed it can be placed at the end of the chache + * aging queue and thus become the first one to be reused when a new element is queried. + * This can be done on a checked out element as well as on an element which is in the cache. + * The cache must be locked for this operation. + * @param self cache + * @param node a llist member in the cached data which is used as queue in the cache */ static inline void lumiera_mrucache_drop (LumieraMruCache self, LList node) @@ -100,7 +117,12 @@ lumiera_mrucache_drop (LumieraMruCache self, LList node) /** - * Remove an element from a mrucache. + * Checkout an element from a cache. + * A checked out element is not under cache control anymore until is gets checked in again. + * The code which checked the element out takes ownership of the element. + * The cache must be locked for this operation. + * @param self cache + * @param node a llist member in the cached data which is used as queue in the cache */ static inline void lumiera_mrucache_checkout (LumieraMruCache self, LList node) @@ -113,7 +135,11 @@ lumiera_mrucache_checkout (LumieraMruCache self, LList node) /** - * destroy the oldest element and return a pointer to it for reuse. + * Destroy the oldest element from the cache and return it. + * This function is used to get a new element by deleting the oldest least used one from the cache. + * The cache must be locked for this operation. + * @param self cache + * @return pointer to the uninitialized memory ready for being reused */ static inline void* lumiera_mrucache_pop (LumieraMruCache self) @@ -134,7 +160,11 @@ lumiera_mrucache_pop (LumieraMruCache self) /** - * destroy and free the nelem oldest elements + * Destroy and free the nelem oldest elements. + * Used to free up resources and memory. + * @param self cache where to free elements. + * @param nelem number of elements wished to be freed + * @return nelem-(numer of elements which got freed), that is 0 if all requested elements got freed */ int lumiera_mrucache_age (LumieraMruCache self, int nelem); From 03ed87051d8c8377a22f8767e61414660efa4349 Mon Sep 17 00:00:00 2001 From: Christian Thaeter Date: Fri, 2 May 2008 23:58:12 +0200 Subject: [PATCH 21/39] fix missing declaration of lumiera_calloc() in safeclib.h --- src/lib/safeclib.h | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/src/lib/safeclib.h b/src/lib/safeclib.h index 2c835c8c2..f486ed056 100644 --- a/src/lib/safeclib.h +++ b/src/lib/safeclib.h @@ -38,6 +38,17 @@ void* lumiera_malloc (size_t sz); +/** + * Allocate cleared memory for an array. + * always succeeds or dies + * @param n number of elements + * @param size memory to be allocated + * @return pointer to the allocated memory + */ +void* +lumiera_calloc (size_t n, size_t size); + + /** * Duplicate a C string. * always succeeds or dies From 94e075d2c2ea02042b809eb48c8635203d9c8f75 Mon Sep 17 00:00:00 2001 From: Christian Thaeter Date: Fri, 2 May 2008 23:59:26 +0200 Subject: [PATCH 22/39] fix: mrucache_drop() must call the element destructor --- src/lib/mrucache.h | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/lib/mrucache.h b/src/lib/mrucache.h index 3edb680f0..c81bd7599 100644 --- a/src/lib/mrucache.h +++ b/src/lib/mrucache.h @@ -35,7 +35,8 @@ /** * Callback function used to destruct/cleanup aged elements. - * shall clean the element sufficiently up to be ready for being freed or reused + * shall clean the element sufficiently up to be ready for being freed or reused, + * this callback function must be reentrant and prepared to be called twice. * @param node the llist node used to link cache elements (will be empty at call) * @return pointer to the begin of the element. */ @@ -113,6 +114,9 @@ lumiera_mrucache_drop (LumieraMruCache self, LList node) REQUIRE (llist_is_member (&self->cache_list, node), "node must be empty or member of cache"); } llist_insert_tail (&self->cache_list, node); + + if (self->destructor_cb) + self->destructor_cb (node); } From 791f42c10474cd26f096c070a0feba51e4c72706 Mon Sep 17 00:00:00 2001 From: Christian Thaeter Date: Sun, 30 Mar 2008 04:31:32 +0200 Subject: [PATCH 23/39] WIP: Low level file management We have 'File' which associates a name with a file and will be used to offer an interface for the application. Posix 'Filehandles' which are managed in a 'Filehandlecache' to utilize a limited number of fd's at optimum. Internally we use 'Filedescriptor's which manage files, mapping and all kinds of metadata. Filedescriptors weak referenced by a registry and freed when the last user vanishes. --- src/backend/Makefile.am | 15 ++- src/backend/file.c | 64 +++++++++++ src/backend/file.h | 90 +++++++++++++++ src/backend/filedescriptor.c | 199 ++++++++++++++++++++++++++++++++++ src/backend/filedescriptor.h | 58 ++++++++++ src/backend/filehandle.h | 51 +++++++++ src/backend/filehandlecache.c | 83 ++++++++++++++ src/backend/filehandlecache.h | 97 +++++++++++++++++ 8 files changed, 649 insertions(+), 8 deletions(-) create mode 100644 src/backend/file.c create mode 100644 src/backend/file.h create mode 100644 src/backend/filedescriptor.c create mode 100644 src/backend/filedescriptor.h create mode 100644 src/backend/filehandle.h create mode 100644 src/backend/filehandlecache.c create mode 100644 src/backend/filehandlecache.h diff --git a/src/backend/Makefile.am b/src/backend/Makefile.am index eb5853d5f..45ce555a8 100644 --- a/src/backend/Makefile.am +++ b/src/backend/Makefile.am @@ -31,12 +31,11 @@ liblumibackend_a_SOURCES = \ # $(liblumibackend_a_srcdir)/filehandlecache.c -noinst_HEADERS += \ - $(liblumibackend_a_srcdir)/mediaaccessfacade.cpp - -# $(liblumibackend_a_srcdir)/backend.h \ -# $(liblumibackend_a_srcdir)/file.h \ -# $(liblumibackend_a_srcdir)/filehandle.h \ -# $(liblumibackend_a_srcdir)/filedescriptor.h \ -# $(liblumibackend_a_srcdir)/filehandlecache.h +noinst_HEADERS += \ + $(liblumibackend_a_srcdir)/mediaaccessfacade.cpp \ + $(liblumibackend_a_srcdir)/backend.h \ + $(liblumibackend_a_srcdir)/file.h \ + $(liblumibackend_a_srcdir)/filehandle.h \ + $(liblumibackend_a_srcdir)/filedescriptor.h \ + $(liblumibackend_a_srcdir)/filehandlecache.h diff --git a/src/backend/file.c b/src/backend/file.c new file mode 100644 index 000000000..7a4aa5050 --- /dev/null +++ b/src/backend/file.c @@ -0,0 +1,64 @@ +/* + file.c - file handling + + Copyright (C) Lumiera.org + 2008, Christian Thaeter + + This program is free software; you can redistribute it and/or + modify it under the terms of the GNU General Public License as + published by the Free Software Foundation; either version 2 of the + License, or (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, write to the Free Software + Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA. +*/ + +#include "lib/mutex.h" +#include "lib/safeclib.h" + +#include "backend/file.h" +#include "backend/filedescriptor.h" + +#include + + +NOBUG_DEFINE_FLAG (lumiera_file); + +LumieraFile +lumiera_file_init (LumieraFile self, const char* name, int flags) +{ + if (!lumiera_filedescriptor_acquire (&self->descriptor, name, flags)) + return NULL; + self->name = lumiera_strndup (name, PATH_MAX); + + return self; +} + +LumieraFile +lumiera_file_destroy (LumieraFile self) +{ + lumiera_reference_destroy (&self->descriptor); + free ((void*)self->name); + return self; +} + + +LumieraFile +lumiera_file_new (const char* name, int flags) +{ + LumieraFile self = lumiera_malloc (sizeof (lumiera_file)); + return lumiera_file_init (self, name, flags); +} + +void +lumiera_file_delete (LumieraFile self) +{ + free (lumiera_file_destroy (self)); +} + diff --git a/src/backend/file.h b/src/backend/file.h new file mode 100644 index 000000000..5018d321a --- /dev/null +++ b/src/backend/file.h @@ -0,0 +1,90 @@ +/* + file.h - interface to files by filename + + Copyright (C) Lumiera.org + 2008, Christian Thaeter + + This program is free software; you can redistribute it and/or + modify it under the terms of the GNU General Public License as + published by the Free Software Foundation; either version 2 of the + License, or (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, write to the Free Software + Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA. +*/ + +#ifndef LUMIERA_FILE_H +#define LUMIERA_FILE_H + + +#include "lib/mutex.h" +#include "lib/llist.h" +#include "lib/error.h" +#include "lib/references.h" + +/** + * @file File management + * Handling Files is splitted into different classes: + * 1. The 'lumiera_file' class which acts as interface to the outside for managing files. + * 'lumiera_file' is addressed by the name of the file. Since files can have more than one name (hardlinks) + * many 'lumiera_file' can point to a single 'lumiera_filedescriptor' + * 2. The 'lumiera_filedescriptor' class which does the real work managing the file in the back. + * 3. Since OS-filehandles are a limited resource we access the lazily as 'lumiera_filehandle' which are + * managed in a 'lumiera_filehandlecache' + */ + +typedef struct lumiera_file_struct lumiera_file; +typedef lumiera_file* LumieraFile; + + +#include "backend/filehandle.h" +#include "backend/filedescriptor.h" + + + +NOBUG_DECLARE_FLAG (lumiera_file); + +#define LUMIERA_FILE_READONLY (O_RDONLY | O_LARGEFILE | O_NOATIME) +#define LUMIERA_FILE_READWRITE (O_RDWR | O_LARGEFILE | O_NOATIME) +#define LUMIERA_FILE_CREATE (O_RDWR | O_LARGEFILE | O_NOATIME | O_CREAT | O_EXCL) +#define LUMIERA_FILE_RECREATE (O_RDWR | O_LARGEFILE | O_NOATIME | O_CREAT | O_TRUNC) + +struct lumiera_file_struct +{ + const char* name; + struct lumiera_reference_struct descriptor; +}; + +/** + * + */ +LumieraFile +lumiera_file_init (LumieraFile self, const char* name, int flags); + +/** + * + */ +LumieraFile +lumiera_file_destroy (LumieraFile self); + +/** + * + */ +LumieraFile +lumiera_file_new (const char* name, int flags); + +/** + * + */ +void +lumiera_file_delete (LumieraFile self); + + +#endif + diff --git a/src/backend/filedescriptor.c b/src/backend/filedescriptor.c new file mode 100644 index 000000000..fdb22ffb3 --- /dev/null +++ b/src/backend/filedescriptor.c @@ -0,0 +1,199 @@ +/* + filedescriptor.c - file handling + + Copyright (C) Lumiera.org + 2008, Christian Thaeter + + This program is free software; you can redistribute it and/or + modify it under the terms of the GNU General Public License as + published by the Free Software Foundation; either version 2 of the + License, or (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, write to the Free Software + Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA. +*/ + +#include "lib/mutex.h" +#include "lib/safeclib.h" +#include "lib/cuckoo.h" + +#include "backend/filedescriptor.h" + +#include +#include +#include +#include + + +NOBUG_DEFINE_FLAG (filedescriptor); + +static Cuckoo registry = NULL; +static lumiera_mutex registry_mutex = {PTHREAD_MUTEX_INITIALIZER}; + + +static size_t +h1 (const void* item, const uint32_t r) +{ + const LumieraFiledescriptor i = (const LumieraFiledescriptor) item; + return i->stat.st_dev^i->stat.st_ino^i->flags^((i->stat.st_dev^i->stat.st_ino^i->flags)>>7)^r; +} + +static size_t +h2 (const void* item, const uint32_t r) +{ + const LumieraFiledescriptor i = (const LumieraFiledescriptor) item; + return i->stat.st_dev^i->stat.st_ino^i->flags^((i->stat.st_dev^i->stat.st_ino^i->flags)>>5)^r; +} + +static size_t +h3 (const void* item, const uint32_t r) +{ + const LumieraFiledescriptor i = (const LumieraFiledescriptor) item; + return i->stat.st_dev^i->stat.st_ino^i->flags^((i->stat.st_dev^i->stat.st_ino^i->flags)>>3)^r; +} + + +static int +cmp (const void* keya, const void* keyb) +{ + const LumieraFiledescriptor a = (const LumieraFiledescriptor) keya; + const LumieraFiledescriptor b = (const LumieraFiledescriptor) keyb; + return a->stat.st_dev == b->stat.st_dev && a->stat.st_ino == b->stat.st_ino && a->flags == b->flags; +} + + +static Cuckoo +lumiera_filedescriptor_registry_new (void) +{ + NOBUG_INIT_FLAG (filedescriptor); + + Cuckoo registry = cuckoo_new (h1, h2, h3, cmp, + sizeof (struct lumiera_reference_struct), + 3); + if (!registry) + LUMIERA_DIE (NO_MEMORY); + + return registry; +} + +/** + * Find existing filedescriptor or create one + * @param descriptor strong reference to be initialized with the filedescriptor, + * a filedescriptor will be deleted when its last string reference gets deleted. + * @param name name of the file + * @param flags opening flags for the filedescriptor + * @return descriptor on success or NULL on error + */ +LumieraReference +lumiera_filedescriptor_acquire (LumieraReference descriptor, const char* name, int flags) +{ + UNCHECKED; + lumiera_mutexacquirer registry_lock; + lumiera_mutexacquirer_init_mutex (®istry_lock, ®istry_mutex, LUMIERA_LOCKED); + + if (!registry) + registry = lumiera_filedescriptor_registry_new (); + + struct lumiera_filedescriptor_struct findme; + findme.flags = flags; + + for (int retry = 0; !retry; ++retry) + { + if (stat (name, &findme.stat) == 0) + break; + else + { + if (errno == ENOENT && flags&O_CREAT) + { + TRACE (filedescriptor, "try creating file: %s", name); + char* slash = lumiera_tmpbuf_strndup (name, PATH_MAX); + + while ((slash = strchr (slash+1, '/'))) + { + *slash = '\0'; + if (mkdir (name, 0777) == -1) + { + LUMIERA_ERROR_SET (filedescriptor, ERRNO); + goto efile; + } + *slash = '/'; + } + + int fd; + fd = creat (name, 0777); + if (fd == -1) + { + LUMIERA_ERROR_SET (filedescriptor, ERRNO); + goto efile; + } + close (fd); + /* will now retry the fstat once */ + } + else + { + LUMIERA_ERROR_SET (filedescriptor, ERRNO); + goto efile; + } + } + } + + /* lookup/create descriptor */ + LumieraReference elem = cuckoo_find (registry, &findme); + + if (!elem) + { + TRACE (filedescriptor, "Descriptor not found"); + + lumiera_reference_strong_init_once (descriptor, + lumiera_filedescriptor_new (&findme), + (void(*)(void*))lumiera_filedescriptor_delete); + + if (!lumiera_reference_get (descriptor)) + goto ecreate; + + lumiera_reference weak_descriptor; + lumiera_reference_weak_init (&weak_descriptor, descriptor); + cuckoo_insert (registry, &weak_descriptor); + } + else + { + lumiera_reference_strong_init (descriptor, elem); + } + lumiera_mutexacquirer_unlock (®istry_lock); + + return descriptor; + + efile: + ecreate: + lumiera_mutexacquirer_unlock (®istry_lock); + return NULL; +} + + +/** + * Allocate a new filedescriptor cloned from a template + * @param template the source filedescriptor + * @return the constrccted filedescriptor + */ +LumieraFiledescriptor +lumiera_filedescriptor_new (LumieraFiledescriptor template) +{ + UNIMPLEMENTED(""); + return NULL; +} + +/** + * delete a filedescriptor and free its resources + * @param descriptor filedescriptor to be freed + */ +void +lumiera_filedescriptor_delete (LumieraFiledescriptor descriptor) +{ + UNIMPLEMENTED(""); +} diff --git a/src/backend/filedescriptor.h b/src/backend/filedescriptor.h new file mode 100644 index 000000000..060839d0a --- /dev/null +++ b/src/backend/filedescriptor.h @@ -0,0 +1,58 @@ +/* + filedescriptor.h - file handling + + Copyright (C) Lumiera.org + 2008, Christian Thaeter + + This program is free software; you can redistribute it and/or + modify it under the terms of the GNU General Public License as + published by the Free Software Foundation; either version 2 of the + License, or (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, write to the Free Software + Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA. +*/ + +#ifndef LUMIERA_FILEDESCRIPTOR_H +#define LUMIERA_FILEDESCRIPTOR_H + +#include "lib/mutex.h" + +#include + +typedef struct lumiera_filedescriptor_struct lumiera_filedescriptor; +typedef lumiera_filedescriptor* LumieraFiledescriptor; + + +#include "backend/filehandle.h" +#include "backend/file.h" + + +struct lumiera_filedescriptor_struct +{ + struct stat stat; /* create after first open, maintained metadata, MUST BE FIRST! */ + int flags; /* open flags, must be masked for reopen */ + lumiera_mutex lock; /* locks operations on this file descriptor */ + unsigned reopened; /* count for reopens 0=not yet opened, 1=first, 2..=reopened*/ + //LumieraFileMap mappings; + //LumieraWriteBuffer writebuffer; +}; + + +LumieraReference +lumiera_filedescriptor_acquire (LumieraReference descriptor, const char* name, int flags); + +LumieraFiledescriptor +lumiera_filedescriptor_new (LumieraFiledescriptor template); + +void +lumiera_filedescriptor_delete (LumieraFiledescriptor self); + + +#endif diff --git a/src/backend/filehandle.h b/src/backend/filehandle.h new file mode 100644 index 000000000..00b4737c9 --- /dev/null +++ b/src/backend/filehandle.h @@ -0,0 +1,51 @@ +/* + filehandle - filehandle management and caching + + Copyright (C) Lumiera.org + 2008, Christian Thaeter + + This program is free software; you can redistribute it and/or + modify it under the terms of the GNU General Public License as + published by the Free Software Foundation; either version 2 of the + License, or (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, write to the Free Software + Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA. +*/ +#ifndef LUMIERA_FILEHANDLE_H +#define LUMIERA_FILEHANDLE_H + +#include "lib/error.h" +#include "lib/llist.h" + +/** + * @file Filehandles + */ + + +/** + * File handles + */ +struct lumiera_filehandle_struct +{ + llist cachenode; + int handle; + //LumieraFile file; + //lumiera_mutex lock; +}; +typedef struct lumiera_filehandle_struct lumiera_filehandle; +typedef lumiera_filehandle* LumieraFilehandle; + +LumieraFilehandle +lumiera_filehandle_new (); + +void* +lumiera_filehandle_destroy_node (LList node); + +#endif diff --git a/src/backend/filehandlecache.c b/src/backend/filehandlecache.c new file mode 100644 index 000000000..98ebdddc9 --- /dev/null +++ b/src/backend/filehandlecache.c @@ -0,0 +1,83 @@ +/* + filehandlecache - filehandle management and caching + + Copyright (C) Lumiera.org + 2008, Christian Thaeter + + This program is free software; you can redistribute it and/or + modify it under the terms of the GNU General Public License as + published by the Free Software Foundation; either version 2 of the + License, or (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, write to the Free Software + Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA. +*/ + +#include "backend/filehandlecache.h" + +NOBUG_DEFINE_FLAG (lumiera_filehandlecache); + +/* errors */ + +LUMIERA_ERROR_DEFINE (FILEHANDLECACHE_NOHANDLE, "No filehandle available"); + + +/**/ + +LumieraFilehandlecache +lumiera_filehandlecache_init (LumieraFilehandlecache self, int max_entries) +{ + lumiera_mrucache_init (&self->cache, lumiera_filehandle_destroy_node); + self->available =max_entries; + lumiera_mutex_init (&self->lock); + return self; +} + +LumieraFilehandlecache +lumiera_filehandlecache_destroy (LumieraFilehandlecache self) +{ + self->available += self->cache.cached; + lumiera_mrucache_destroy (&self->cache); + lumiera_mutex_destroy (&self->lock); + return self; +} + + +LumieraFilehandle +lumiera_filehandlecache_filehandle_acquire (LumieraFilehandlecache self) +{ + LumieraFilehandle ret; + LUMIERA_MUTEX_SECTION (&self->lock) + { + if (self->available <= 0 && self->cache.cached) + { + /* pop a filehandle from cache */ + ret = lumiera_mrucache_pop (&self->cache); + if (self->available < 0) + /* try to free overallocated filehandles */ + self->available -= self->available + lumiera_mrucache_age (&self->cache, -self->available); + } + else + { + /* allocate new filehandle if we are below the limit or no cached handles where available (overallocating) */ + ret = lumiera_filehandle_new (); + if (!ret) + LUMIERA_ERROR_SET (lumiera_filehandlecache, FILEHANDLECACHE_NOHANDLE); + else + --self->available; + } + } + return ret; +} + +void +lumiera_filehandlecache_add (LumieraFilehandlecache self, LumieraFilehandle handle) +{ + UNIMPLEMENTED (""); +} diff --git a/src/backend/filehandlecache.h b/src/backend/filehandlecache.h new file mode 100644 index 000000000..40386defe --- /dev/null +++ b/src/backend/filehandlecache.h @@ -0,0 +1,97 @@ +/* + filehandlecache - filehandle management and caching + + Copyright (C) Lumiera.org + 2008, Christian Thaeter + + This program is free software; you can redistribute it and/or + modify it under the terms of the GNU General Public License as + published by the Free Software Foundation; either version 2 of the + License, or (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, write to the Free Software + Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA. +*/ +#ifndef LUMIERA_FILEHANDLECACHE_H +#define LUMIERA_FILEHANDLECACHE_H + +#include "lib/error.h" +#include "lib/mrucache.h" +#include "lib/mutex.h" + +#include "backend/filehandle.h" + +/** + * @file Filehandle management and caching + * The number of filehandles a program can held open is usually limited, since we want to support + * using a less limited number of files and closing/opening for each operation is expensive, we + * provide a cache to keep the most frequent used files open and gracefully close/recycle unused filehandles. + */ + + + +/* + File handle cache manages file handles + */ +struct lumiera_filehandlecache_struct +{ + lumiera_mrucache cache; + int available; + lumiera_mutex lock; +}; +typedef struct lumiera_filehandlecache_struct lumiera_filehandlecache; +typedef lumiera_filehandlecache* LumieraFilehandlecache; + + +/** + * Initializes a filehandle cache. + * @param self pointer to the cache to be initialized + * @param max_entries number how much filehandles shall be managed + * @return self as given + * The number of elements the cache can hold is static and should be + * determined by sysconf (_SC_OPEN_MAX) minus some safety margin. + */ +LumieraFilehandlecache +lumiera_filehandlecache_init (LumieraFilehandlecache self, int max_entries); + +/** + * Destroy a filehandle cache. + * @param self the cache to be destroyed + * @return self as given to the now uninitialized! cache + * No filehandles in the cache must be locked, this would be a fatal error. + * The handles are closed automatically. + */ +LumieraFilehandlecache +lumiera_filehandlecache_destroy (LumieraFilehandlecache self); + +/** + * Get a fresh filehandle. + * @param self pointer to the cache + * @return the new filehandle + * one must call lumiera_mutexacquirer_unlock (&lock) to free the lock on the handle when done + */ +LumieraFilehandle +lumiera_filehandlecache_filehandle_acquire (LumieraFilehandlecache self); + +/** + * Add filehande back to cache. + * @param self pointer to the cache + * @param handle filehandle to be put back + */ +void +lumiera_filehandlecache_add (LumieraFilehandlecache self, LumieraFilehandle handle); + +/** + * + */ +LumieraFilehandle +lumiera_filehandlecache_remove (LumieraFilehandlecache self, int fd); + + +#endif From 7a266f22ba1dc41fd0519f8a47b86570cb891582 Mon Sep 17 00:00:00 2001 From: Christian Thaeter Date: Mon, 31 Mar 2008 02:28:25 +0200 Subject: [PATCH 24/39] integrate filedescriptor tests --- tests/20filedescriptor.tests | 20 ++++++++ tests/Makefile.am | 5 ++ tests/backend/test-filedescriptors.c | 70 ++++++++++++++++++++++++++++ 3 files changed, 95 insertions(+) create mode 100644 tests/20filedescriptor.tests create mode 100644 tests/backend/test-filedescriptors.c diff --git a/tests/20filedescriptor.tests b/tests/20filedescriptor.tests new file mode 100644 index 000000000..703a3351e --- /dev/null +++ b/tests/20filedescriptor.tests @@ -0,0 +1,20 @@ + +TESTING "Filedescriptor management" ./test-filedescriptors + +echo testdata > ,tmp_testfile + +TEST "acquire existing file" acquire_existing < + + This program is free software; you can redistribute it and/or + modify it under the terms of the GNU General Public License as + published by the Free Software Foundation; either version 2 of the + License, or (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, write to the Free Software + Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA. +*/ +#define _GNU_SOURCE + +#include "lib/references.h" +#include "lib/safeclib.h" + +#include "backend/filedescriptor.h" + +#include "tests/test.h" + +TESTS_BEGIN + + +TEST ("acquire_existing") +{ + lumiera_reference descriptor; + if (lumiera_filedescriptor_acquire (&descriptor, ",tmp_testfile", LUMIERA_FILE_READONLY)) + { + lumiera_reference_destroy (&descriptor); + return 0; + } + else + return 1; +} + +TEST ("acquire_create") +{ + lumiera_reference descriptor; + if (lumiera_filedescriptor_acquire (&descriptor, ",tmp_testfile", LUMIERA_FILE_CREATE)) + { + lumiera_reference_destroy (&descriptor); + return 0; + } + else + return 1; +} + +TEST ("acquire_create_dir") +{ + lumiera_reference descriptor; + if (lumiera_filedescriptor_acquire (&descriptor, ",tmp_testdir/,tmp_testfile", LUMIERA_FILE_CREATE)) + { + lumiera_reference_destroy (&descriptor); + return 0; + } + else + return 1; +} + + +TESTS_END From e2e94884badc0e58606f0841dfd91d3368b051f6 Mon Sep 17 00:00:00 2001 From: Christian Thaeter Date: Tue, 1 Apr 2008 19:20:17 +0200 Subject: [PATCH 25/39] some refinements to the todo wiki --- wiki/todo.html | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/wiki/todo.html b/wiki/todo.html index d6caf66be..e66ec929f 100644 --- a/wiki/todo.html +++ b/wiki/todo.html @@ -3344,7 +3344,11 @@ Someone might pickup a Doxygen-Maintainer. This is not writing actual documentat 
Use the backend, gmerlin/avdecoder and a Player widget to playback videos. This should be a small application which can open and playback many videos from one instance in parallel. Later on scrubbing for this videos might be supported.
 
-The idea here is to show and measure backend and scheduler performance and beeing a proof of concept for the design. It will run independent of the Proc layer but needs quite some work on the backend to be done.
+The idea here is to show and measure backend and scheduler performance + and beeing a proof of concept for the design. It will run + independent of the Proc layer but needs quite some work on the + backend to be done. +
From 52868388866c6c038cbeba9286092ee65d1f3ed6 Mon Sep 17 00:00:00 2001 From: Christian Thaeter Date: Tue, 1 Apr 2008 22:32:35 +0200 Subject: [PATCH 26/39] filedescriptor handling basics working, including some tests It is now possible to acquire/release filedescriptors for existing files and files to be created. Filedescriptors are managed in a registry by refcounts, they get automatically destructed when the last user releases them. --- src/backend/filedescriptor.c | 123 ++++++++++++++++++--------- src/backend/filedescriptor.h | 14 ++- tests/20filedescriptor.tests | 8 +- tests/backend/test-filedescriptors.c | 35 +++++--- 4 files changed, 125 insertions(+), 55 deletions(-) diff --git a/src/backend/filedescriptor.c b/src/backend/filedescriptor.c index fdb22ffb3..6db2882c9 100644 --- a/src/backend/filedescriptor.c +++ b/src/backend/filedescriptor.c @@ -33,28 +33,35 @@ NOBUG_DEFINE_FLAG (filedescriptor); +/* + Filedescriptor registry + + This registry stores all acquired filedescriptors for lookup, they will be freed when not referenced anymore. + */ static Cuckoo registry = NULL; static lumiera_mutex registry_mutex = {PTHREAD_MUTEX_INITIALIZER}; - +/* + * setup hashing and compare functions for cuckoo hashing + */ static size_t h1 (const void* item, const uint32_t r) { - const LumieraFiledescriptor i = (const LumieraFiledescriptor) item; + const LumieraFiledescriptor i = *(const LumieraFiledescriptor*)item; return i->stat.st_dev^i->stat.st_ino^i->flags^((i->stat.st_dev^i->stat.st_ino^i->flags)>>7)^r; } static size_t h2 (const void* item, const uint32_t r) { - const LumieraFiledescriptor i = (const LumieraFiledescriptor) item; + const LumieraFiledescriptor i = *(const LumieraFiledescriptor*)item; return i->stat.st_dev^i->stat.st_ino^i->flags^((i->stat.st_dev^i->stat.st_ino^i->flags)>>5)^r; } static size_t h3 (const void* item, const uint32_t r) { - const LumieraFiledescriptor i = (const LumieraFiledescriptor) item; + const LumieraFiledescriptor i = *(const LumieraFiledescriptor*)item; return i->stat.st_dev^i->stat.st_ino^i->flags^((i->stat.st_dev^i->stat.st_ino^i->flags)>>3)^r; } @@ -62,19 +69,18 @@ h3 (const void* item, const uint32_t r) static int cmp (const void* keya, const void* keyb) { - const LumieraFiledescriptor a = (const LumieraFiledescriptor) keya; - const LumieraFiledescriptor b = (const LumieraFiledescriptor) keyb; + const LumieraFiledescriptor a = *(const LumieraFiledescriptor*)keya; + const LumieraFiledescriptor b = *(const LumieraFiledescriptor*)keyb; return a->stat.st_dev == b->stat.st_dev && a->stat.st_ino == b->stat.st_ino && a->flags == b->flags; } - static Cuckoo lumiera_filedescriptor_registry_new (void) { NOBUG_INIT_FLAG (filedescriptor); Cuckoo registry = cuckoo_new (h1, h2, h3, cmp, - sizeof (struct lumiera_reference_struct), + sizeof (LumieraFiledescriptor), 3); if (!registry) LUMIERA_DIE (NO_MEMORY); @@ -82,6 +88,12 @@ lumiera_filedescriptor_registry_new (void) return registry; } +static void +lumiera_filedescriptor_registry_delete (void) +{ + cuckoo_free (registry); +} + /** * Find existing filedescriptor or create one * @param descriptor strong reference to be initialized with the filedescriptor, @@ -90,34 +102,40 @@ lumiera_filedescriptor_registry_new (void) * @param flags opening flags for the filedescriptor * @return descriptor on success or NULL on error */ -LumieraReference -lumiera_filedescriptor_acquire (LumieraReference descriptor, const char* name, int flags) +LumieraFiledescriptor +lumiera_filedescriptor_acquire (const char* name, int flags) { + TRACE (filedescriptor); UNCHECKED; lumiera_mutexacquirer registry_lock; lumiera_mutexacquirer_init_mutex (®istry_lock, ®istry_mutex, LUMIERA_LOCKED); if (!registry) - registry = lumiera_filedescriptor_registry_new (); + { + registry = lumiera_filedescriptor_registry_new (); + atexit (lumiera_filedescriptor_registry_delete); + } - struct lumiera_filedescriptor_struct findme; - findme.flags = flags; + lumiera_filedescriptor fdesc; + fdesc.flags = flags; for (int retry = 0; !retry; ++retry) { - if (stat (name, &findme.stat) == 0) + if (stat (name, &fdesc.stat) == 0) break; else { if (errno == ENOENT && flags&O_CREAT) { - TRACE (filedescriptor, "try creating file: %s", name); - char* slash = lumiera_tmpbuf_strndup (name, PATH_MAX); + INFO (filedescriptor, "try creating file: %s %d", name, PATH_MAX); + char* dir = lumiera_tmpbuf_strndup (name, PATH_MAX); + char* slash = dir; while ((slash = strchr (slash+1, '/'))) { *slash = '\0'; - if (mkdir (name, 0777) == -1) + INFO (filedescriptor, "try creating dir: %s", dir); + if (mkdir (dir, 0777) == -1) { LUMIERA_ERROR_SET (filedescriptor, ERRNO); goto efile; @@ -133,41 +151,37 @@ lumiera_filedescriptor_acquire (LumieraReference descriptor, const char* name, i goto efile; } close (fd); - /* will now retry the fstat once */ - } - else - { - LUMIERA_ERROR_SET (filedescriptor, ERRNO); - goto efile; + continue; /* will now retry the fstat once */ } } + /* finally, no luck */ + LUMIERA_ERROR_SET (filedescriptor, ERRNO); + goto efile; } /* lookup/create descriptor */ - LumieraReference elem = cuckoo_find (registry, &findme); + LumieraFiledescriptor dest = &fdesc; + LumieraFiledescriptor* found = cuckoo_find (registry, &dest); - if (!elem) + if (!found) { TRACE (filedescriptor, "Descriptor not found"); - lumiera_reference_strong_init_once (descriptor, - lumiera_filedescriptor_new (&findme), - (void(*)(void*))lumiera_filedescriptor_delete); - - if (!lumiera_reference_get (descriptor)) + dest = lumiera_filedescriptor_new (&fdesc); + if (!dest) goto ecreate; - lumiera_reference weak_descriptor; - lumiera_reference_weak_init (&weak_descriptor, descriptor); - cuckoo_insert (registry, &weak_descriptor); + cuckoo_insert (registry, &dest); } else { - lumiera_reference_strong_init (descriptor, elem); + TRACE (filedescriptor, "Descriptor already existing"); + dest = *found; + ++dest->refcount; } - lumiera_mutexacquirer_unlock (®istry_lock); - return descriptor; + lumiera_mutexacquirer_unlock (®istry_lock); + return dest; efile: ecreate: @@ -175,6 +189,13 @@ lumiera_filedescriptor_acquire (LumieraReference descriptor, const char* name, i return NULL; } +void +lumiera_filedescriptor_release (LumieraFiledescriptor self) +{ + TRACE (filedescriptor); + if (!--self->refcount) + lumiera_filedescriptor_delete (self); +} /** * Allocate a new filedescriptor cloned from a template @@ -184,8 +205,17 @@ lumiera_filedescriptor_acquire (LumieraReference descriptor, const char* name, i LumieraFiledescriptor lumiera_filedescriptor_new (LumieraFiledescriptor template) { - UNIMPLEMENTED(""); - return NULL; + LumieraFiledescriptor self = lumiera_malloc (sizeof (lumiera_filedescriptor)); + TRACE (filedescriptor, "at %p", self); + + self->stat = template->stat; + + self->flags = template->flags; + lumiera_mutex_init (&self->lock); + self->reopened = 0; + self->refcount = 1; + + return self; } /** @@ -193,7 +223,20 @@ lumiera_filedescriptor_new (LumieraFiledescriptor template) * @param descriptor filedescriptor to be freed */ void -lumiera_filedescriptor_delete (LumieraFiledescriptor descriptor) +lumiera_filedescriptor_delete (LumieraFiledescriptor self) { - UNIMPLEMENTED(""); + TRACE (filedescriptor, "%p", self); + lumiera_mutexacquirer registry_lock; + lumiera_mutexacquirer_init_mutex (®istry_lock, ®istry_mutex, LUMIERA_LOCKED); + + REQUIRE (self->refcount == 0); + + cuckoo_remove (registry, cuckoo_find (registry, &self)); + + TODO ("destruct other members (WIP)"); + + lumiera_mutex_destroy (&self->lock); + free (self); + + lumiera_mutexacquirer_unlock (®istry_lock); } diff --git a/src/backend/filedescriptor.h b/src/backend/filedescriptor.h index 060839d0a..59c4eacef 100644 --- a/src/backend/filedescriptor.h +++ b/src/backend/filedescriptor.h @@ -24,7 +24,10 @@ #include "lib/mutex.h" + +#include #include +#include typedef struct lumiera_filedescriptor_struct lumiera_filedescriptor; typedef lumiera_filedescriptor* LumieraFiledescriptor; @@ -39,14 +42,19 @@ struct lumiera_filedescriptor_struct struct stat stat; /* create after first open, maintained metadata, MUST BE FIRST! */ int flags; /* open flags, must be masked for reopen */ lumiera_mutex lock; /* locks operations on this file descriptor */ - unsigned reopened; /* count for reopens 0=not yet opened, 1=first, 2..=reopened*/ + unsigned reopened; /* count for reopens 0=not yet opened, 1=first, 2..=reopened */ + unsigned refcount; /* reference counter, all users sans registry */ + //LumieraFileMap mappings; //LumieraWriteBuffer writebuffer; }; -LumieraReference -lumiera_filedescriptor_acquire (LumieraReference descriptor, const char* name, int flags); +LumieraFiledescriptor +lumiera_filedescriptor_acquire (const char* name, int flags); + +void +lumiera_filedescriptor_release (LumieraFiledescriptor self); LumieraFiledescriptor lumiera_filedescriptor_new (LumieraFiledescriptor template); diff --git a/tests/20filedescriptor.tests b/tests/20filedescriptor.tests index 703a3351e..78639e7b8 100644 --- a/tests/20filedescriptor.tests +++ b/tests/20filedescriptor.tests @@ -4,17 +4,23 @@ TESTING "Filedescriptor management" ./test-filedescriptors echo testdata > ,tmp_testfile TEST "acquire existing file" acquire_existing < Date: Fri, 4 Apr 2008 08:01:59 +0200 Subject: [PATCH 27/39] add nobug resource tracking to filedescriptors --- src/backend/filedescriptor.c | 8 ++++++++ src/backend/filedescriptor.h | 3 +++ 2 files changed, 11 insertions(+) diff --git a/src/backend/filedescriptor.c b/src/backend/filedescriptor.c index 6db2882c9..51b536e56 100644 --- a/src/backend/filedescriptor.c +++ b/src/backend/filedescriptor.c @@ -214,6 +214,9 @@ lumiera_filedescriptor_new (LumieraFiledescriptor template) lumiera_mutex_init (&self->lock); self->reopened = 0; self->refcount = 1; + self->handle = 0; + + RESOURCE_ANNOUNCE (filedescriptor, "mutex", "filedescriptor", self, self->rh); return self; } @@ -231,10 +234,15 @@ lumiera_filedescriptor_delete (LumieraFiledescriptor self) REQUIRE (self->refcount == 0); + RESOURCE_FORGET (filedescriptor, self->rh); + cuckoo_remove (registry, cuckoo_find (registry, &self)); TODO ("destruct other members (WIP)"); + + TODO ("release filehandle"); + lumiera_mutex_destroy (&self->lock); free (self); diff --git a/src/backend/filedescriptor.h b/src/backend/filedescriptor.h index 59c4eacef..48d0cd21c 100644 --- a/src/backend/filedescriptor.h +++ b/src/backend/filedescriptor.h @@ -45,8 +45,11 @@ struct lumiera_filedescriptor_struct unsigned reopened; /* count for reopens 0=not yet opened, 1=first, 2..=reopened */ unsigned refcount; /* reference counter, all users sans registry */ + LumieraFilehandle handle; //LumieraFileMap mappings; //LumieraWriteBuffer writebuffer; + + RESOURCE_HANDLE (rh); }; From b11ff40fbe33846b3f12f38bac35bf00029a85f8 Mon Sep 17 00:00:00 2001 From: Christian Thaeter Date: Mon, 7 Apr 2008 05:51:32 +0200 Subject: [PATCH 28/39] Filedescriptor updates * make registry initialization singleton, no parameters. * remove registry init from all functions, has to be called explicitly. * some nobug improvements --- src/backend/filedescriptor.c | 36 ++++++++++++++++++------------------ src/backend/filedescriptor.h | 6 +++++- 2 files changed, 23 insertions(+), 19 deletions(-) diff --git a/src/backend/filedescriptor.c b/src/backend/filedescriptor.c index 51b536e56..0a3d6e048 100644 --- a/src/backend/filedescriptor.c +++ b/src/backend/filedescriptor.c @@ -31,7 +31,7 @@ #include -NOBUG_DEFINE_FLAG (filedescriptor); +NOBUG_DEFINE_FLAG_PARENT (filedescriptor, file_all); /* Filedescriptor registry @@ -74,26 +74,28 @@ cmp (const void* keya, const void* keyb) return a->stat.st_dev == b->stat.st_dev && a->stat.st_ino == b->stat.st_ino && a->flags == b->flags; } -static Cuckoo -lumiera_filedescriptor_registry_new (void) +void +lumiera_filedescriptor_registry_init (void) { NOBUG_INIT_FLAG (filedescriptor); + TRACE (filedescriptor); - Cuckoo registry = cuckoo_new (h1, h2, h3, cmp, - sizeof (LumieraFiledescriptor), - 3); + registry = cuckoo_new (h1, h2, h3, cmp, + sizeof (LumieraFiledescriptor), + 3); if (!registry) LUMIERA_DIE (NO_MEMORY); - - return registry; } -static void -lumiera_filedescriptor_registry_delete (void) +void +lumiera_filedescriptor_registry_destroy (void) { + TRACE (filedescriptor); + REQUIRE (!cuckoo_nelements (registry)); cuckoo_free (registry); } + /** * Find existing filedescriptor or create one * @param descriptor strong reference to be initialized with the filedescriptor, @@ -106,18 +108,13 @@ LumieraFiledescriptor lumiera_filedescriptor_acquire (const char* name, int flags) { TRACE (filedescriptor); + REQUIRE (registry, "not initialized"); UNCHECKED; lumiera_mutexacquirer registry_lock; lumiera_mutexacquirer_init_mutex (®istry_lock, ®istry_mutex, LUMIERA_LOCKED); - if (!registry) - { - registry = lumiera_filedescriptor_registry_new (); - atexit (lumiera_filedescriptor_registry_delete); - } - lumiera_filedescriptor fdesc; - fdesc.flags = flags; + fdesc.flags = flags&LUMIERA_FILE_MASK; for (int retry = 0; !retry; ++retry) { @@ -216,7 +213,10 @@ lumiera_filedescriptor_new (LumieraFiledescriptor template) self->refcount = 1; self->handle = 0; - RESOURCE_ANNOUNCE (filedescriptor, "mutex", "filedescriptor", self, self->rh); + const char* type = "mutex"; + const char* name = "filedescriptor"; + + RESOURCE_ANNOUNCE (filedescriptor, type, name, self, self->rh); return self; } diff --git a/src/backend/filedescriptor.h b/src/backend/filedescriptor.h index 48d0cd21c..817e8bc13 100644 --- a/src/backend/filedescriptor.h +++ b/src/backend/filedescriptor.h @@ -52,6 +52,11 @@ struct lumiera_filedescriptor_struct RESOURCE_HANDLE (rh); }; +void +lumiera_filedescriptor_registry_init (void); + +void +lumiera_filedescriptor_registry_destroy (void); LumieraFiledescriptor lumiera_filedescriptor_acquire (const char* name, int flags); @@ -65,5 +70,4 @@ lumiera_filedescriptor_new (LumieraFiledescriptor template); void lumiera_filedescriptor_delete (LumieraFiledescriptor self); - #endif From 232afe77e9aefb10cf9832aa678aa2e4b11f620a Mon Sep 17 00:00:00 2001 From: Christian Thaeter Date: Mon, 7 Apr 2008 06:47:57 +0200 Subject: [PATCH 29/39] Filehandling * add 'backend' for backend global initialization/destruction and common functions. * WIP: 'file' acts as interface to named files, provides posix filehandles with and acquire/release function pair. * 'filehandle' manages posix filehandles internally, refcounted. * WIP: 'filehandlecache' keeps unused filehandles in a MRU cache * WIP: test for filehandle management * improved filedescriptor test --- src/backend/backend.c | 53 ++++++++++++++ src/backend/backend.h | 36 ++++++++++ src/backend/file.c | 53 ++++++++++++-- src/backend/file.h | 17 +++-- src/backend/filedescriptor.c | 4 ++ src/backend/filehandle.c | 70 ++++++++++++++++++ src/backend/filehandle.h | 16 +++-- src/backend/filehandlecache.c | 102 +++++++++++++++++++++------ src/backend/filehandlecache.h | 41 ++++++----- tests/20filehandle.tests | 7 ++ tests/Makefile.am | 8 +-- tests/backend/test-filedescriptors.c | 10 ++- tests/backend/test-filehandles.c | 58 +++++++++++++++ 13 files changed, 415 insertions(+), 60 deletions(-) create mode 100644 src/backend/backend.c create mode 100644 src/backend/backend.h create mode 100644 src/backend/filehandle.c create mode 100644 tests/20filehandle.tests create mode 100644 tests/backend/test-filehandles.c diff --git a/src/backend/backend.c b/src/backend/backend.c new file mode 100644 index 000000000..024f7e6f8 --- /dev/null +++ b/src/backend/backend.c @@ -0,0 +1,53 @@ +/* + backend - common lumiera backend things + + Copyright (C) Lumiera.org + 2008, Christian Thaeter + + This program is free software; you can redistribute it and/or + modify it under the terms of the GNU General Public License as + published by the Free Software Foundation; either version 2 of the + License, or (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, write to the Free Software + Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA. +*/ + +#include "backend/backend.h" +#include "backend/filehandlecache.h" +#include "backend/filedescriptor.h" + +//NOBUG_DEFINE_FLAG_PARENT (backend, lumiera); TODO +NOBUG_DEFINE_FLAG (backend); +NOBUG_DEFINE_FLAG_PARENT (file_all, backend); + +int +lumiera_backend_init (void) +{ + NOBUG_INIT_FLAG (backend); + NOBUG_INIT_FLAG (file_all); + NOBUG_INIT_FLAG (file); + TRACE (backend); + lumiera_filedescriptor_registry_init (); + + int max_entries = 10; TODO("determine by sysconf (_SC_OPEN_MAX) minus some (big) safety margin " + "add some override to run tests with few filehandles"); + + lumiera_filehandlecache_new (max_entries); + + return 0; +} + +void +lumiera_backend_destroy (void) +{ + TRACE (backend); + lumiera_filedescriptor_registry_destroy (); + lumiera_filehandlecache_delete (); +} diff --git a/src/backend/backend.h b/src/backend/backend.h new file mode 100644 index 000000000..bd4c86815 --- /dev/null +++ b/src/backend/backend.h @@ -0,0 +1,36 @@ +/* + backend - common lumiera backend things + + Copyright (C) Lumiera.org + 2008, Christian Thaeter + + This program is free software; you can redistribute it and/or + modify it under the terms of the GNU General Public License as + published by the Free Software Foundation; either version 2 of the + License, or (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, write to the Free Software + Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA. +*/ +#ifndef LUMIERA_BACKEND_H +#define LUMIERA_BACKEND_H + +#include + +NOBUG_DECLARE_FLAG (backend); +NOBUG_DECLARE_FLAG (file_all); +NOBUG_DECLARE_FLAG (file); + +int +lumiera_backend_init (void); + +void +lumiera_backend_destroy (void); + +#endif diff --git a/src/backend/file.c b/src/backend/file.c index 7a4aa5050..e9cd208dc 100644 --- a/src/backend/file.c +++ b/src/backend/file.c @@ -23,17 +23,18 @@ #include "lib/safeclib.h" #include "backend/file.h" +#include "backend/filehandlecache.h" #include "backend/filedescriptor.h" #include - -NOBUG_DEFINE_FLAG (lumiera_file); +NOBUG_DEFINE_FLAG_PARENT (file, file_all); LumieraFile lumiera_file_init (LumieraFile self, const char* name, int flags) { - if (!lumiera_filedescriptor_acquire (&self->descriptor, name, flags)) + TRACE (file); + if (!(self->descriptor = lumiera_filedescriptor_acquire (name, flags))) return NULL; self->name = lumiera_strndup (name, PATH_MAX); @@ -43,7 +44,8 @@ lumiera_file_init (LumieraFile self, const char* name, int flags) LumieraFile lumiera_file_destroy (LumieraFile self) { - lumiera_reference_destroy (&self->descriptor); + TRACE (file); + lumiera_filedescriptor_release (self->descriptor); free ((void*)self->name); return self; } @@ -52,6 +54,7 @@ lumiera_file_destroy (LumieraFile self) LumieraFile lumiera_file_new (const char* name, int flags) { + TRACE (file); LumieraFile self = lumiera_malloc (sizeof (lumiera_file)); return lumiera_file_init (self, name, flags); } @@ -59,6 +62,48 @@ lumiera_file_new (const char* name, int flags) void lumiera_file_delete (LumieraFile self) { + TRACE (file); free (lumiera_file_destroy (self)); } + +int +lumiera_file_handle_acquire (LumieraFile self) +{ + TRACE (file); + REQUIRE (self); + REQUIRE (self->descriptor); + REQUIRE (fhcache); + + LUMIERA_MUTEX_SECTION (file, self->descriptor->rh, &self->descriptor->lock) + { + if (!self->descriptor->handle) + /* no handle yet, get a new one */ + lumiera_filehandlecache_handle_acquire (fhcache, self->descriptor); + else + lumiera_filehandlecache_checkout (fhcache, self->descriptor->handle); + + if (self->descriptor->handle->fd == -1) + { + TODO ("stat handling/update"); + self->descriptor->handle->fd = open (self->name, self->descriptor->flags); + if (self->descriptor->handle->fd == -1) + { + LUMIERA_ERROR_SET (file, ERRNO); + } + } + } + + return self->descriptor->handle->fd; +} + +void +lumiera_file_handle_release (LumieraFile self) +{ + TRACE (file); + + LUMIERA_MUTEX_SECTION (file, self->descriptor->rh, &self->descriptor->lock) + { + lumiera_filehandlecache_checkin (fhcache, self->descriptor->handle); + } +} diff --git a/src/backend/file.h b/src/backend/file.h index 5018d321a..60c35447c 100644 --- a/src/backend/file.h +++ b/src/backend/file.h @@ -26,7 +26,8 @@ #include "lib/mutex.h" #include "lib/llist.h" #include "lib/error.h" -#include "lib/references.h" + +NOBUG_DECLARE_FLAG (file); /** * @file File management @@ -47,18 +48,18 @@ typedef lumiera_file* LumieraFile; #include "backend/filedescriptor.h" - -NOBUG_DECLARE_FLAG (lumiera_file); - #define LUMIERA_FILE_READONLY (O_RDONLY | O_LARGEFILE | O_NOATIME) #define LUMIERA_FILE_READWRITE (O_RDWR | O_LARGEFILE | O_NOATIME) #define LUMIERA_FILE_CREATE (O_RDWR | O_LARGEFILE | O_NOATIME | O_CREAT | O_EXCL) #define LUMIERA_FILE_RECREATE (O_RDWR | O_LARGEFILE | O_NOATIME | O_CREAT | O_TRUNC) +/* creat and excl flags will be masked out for descriptor lookup */ +#define LUMIERA_FILE_MASK ~(O_CREAT | O_EXCL) + struct lumiera_file_struct { const char* name; - struct lumiera_reference_struct descriptor; + LumieraFiledescriptor descriptor; }; /** @@ -86,5 +87,11 @@ void lumiera_file_delete (LumieraFile self); +int +lumiera_file_handle_acquire (LumieraFile self); + +void +lumiera_file_handle_release (LumieraFile self); + #endif diff --git a/src/backend/filedescriptor.c b/src/backend/filedescriptor.c index 0a3d6e048..be166d9c7 100644 --- a/src/backend/filedescriptor.c +++ b/src/backend/filedescriptor.c @@ -23,7 +23,10 @@ #include "lib/safeclib.h" #include "lib/cuckoo.h" +#include "backend/file.h" #include "backend/filedescriptor.h" +#include "backend/filehandle.h" +#include "backend/filehandlecache.h" #include #include @@ -141,6 +144,7 @@ lumiera_filedescriptor_acquire (const char* name, int flags) } int fd; + INFO (filedescriptor, "try creating file: %s", name); fd = creat (name, 0777); if (fd == -1) { diff --git a/src/backend/filehandle.c b/src/backend/filehandle.c new file mode 100644 index 000000000..60d54cf82 --- /dev/null +++ b/src/backend/filehandle.c @@ -0,0 +1,70 @@ +/* + filehandle - filehandle management and caching + + Copyright (C) Lumiera.org + 2008, Christian Thaeter + + This program is free software; you can redistribute it and/or + modify it under the terms of the GNU General Public License as + published by the Free Software Foundation; either version 2 of the + License, or (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, write to the Free Software + Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA. +*/ + +#include "lib/llist.h" +#include "lib/safeclib.h" + +#include "backend/file.h" +#include "backend/filehandle.h" +#include "backend/filedescriptor.h" + +#include + +NOBUG_DEFINE_FLAG_PARENT (filehandle, file_all); + +LumieraFilehandle +lumiera_filehandle_new () +{ + LumieraFilehandle self = lumiera_malloc (sizeof (lumiera_filehandle)); + TRACE (filehandle, "%p", self); + + llist_init (&self->cachenode); + self->fd = -1; + self->use_cnt = 1; + self->descriptor = NULL; + + return self; +} + + +void* +lumiera_filehandle_destroy_node (LList node) +{ + TRACE (filehandle); + REQUIRE (llist_is_empty (node)); + LumieraFilehandle self = (LumieraFilehandle)node; + REQUIRE (self->use_cnt == 0); + + if (self->fd >= 0) + close (self->fd); + self->fd = -1; + self->descriptor = NULL; + return self; +} + + +int +lumiera_filehandle_get (LumieraFilehandle self) +{ + REQUIRE (self->descriptor); + return -1; +} + diff --git a/src/backend/filehandle.h b/src/backend/filehandle.h index 00b4737c9..54e4d6936 100644 --- a/src/backend/filehandle.h +++ b/src/backend/filehandle.h @@ -24,6 +24,11 @@ #include "lib/error.h" #include "lib/llist.h" +typedef struct lumiera_filehandle_struct lumiera_filehandle; +typedef lumiera_filehandle* LumieraFilehandle; + +#include "backend/filedescriptor.h" + /** * @file Filehandles */ @@ -35,16 +40,17 @@ struct lumiera_filehandle_struct { llist cachenode; - int handle; - //LumieraFile file; - //lumiera_mutex lock; + int fd; + unsigned use_cnt; + LumieraFiledescriptor descriptor; }; -typedef struct lumiera_filehandle_struct lumiera_filehandle; -typedef lumiera_filehandle* LumieraFilehandle; LumieraFilehandle lumiera_filehandle_new (); +int +lumiera_filehandle_get (LumieraFilehandle self); + void* lumiera_filehandle_destroy_node (LList node); diff --git a/src/backend/filehandlecache.c b/src/backend/filehandlecache.c index 98ebdddc9..c629f765b 100644 --- a/src/backend/filehandlecache.c +++ b/src/backend/filehandlecache.c @@ -19,41 +19,60 @@ Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA. */ +#include "lib/safeclib.h" + +#include "backend/file.h" #include "backend/filehandlecache.h" -NOBUG_DEFINE_FLAG (lumiera_filehandlecache); +NOBUG_DEFINE_FLAG_PARENT (filehandlecache, file_all); /* errors */ LUMIERA_ERROR_DEFINE (FILEHANDLECACHE_NOHANDLE, "No filehandle available"); -/**/ +LumieraFilehandlecache fhcache = NULL; -LumieraFilehandlecache -lumiera_filehandlecache_init (LumieraFilehandlecache self, int max_entries) + +void +lumiera_filehandlecache_new (int max_entries) { - lumiera_mrucache_init (&self->cache, lumiera_filehandle_destroy_node); - self->available =max_entries; - lumiera_mutex_init (&self->lock); - return self; + if (!fhcache) + { + NOBUG_INIT_FLAG (filehandlecache); + fhcache = lumiera_malloc (sizeof (lumiera_filehandlecache)); + lumiera_mrucache_init (&fhcache->cache, lumiera_filehandle_destroy_node); + fhcache->available = max_entries; + fhcache->checked_out = 0; + lumiera_mutex_init (&fhcache->lock); + RESOURCE_ANNOUNCE (filehandlecache, "mutex", "filehandlecache", fhcache, fhcache->rh); + } + else + NOTREACHED; } -LumieraFilehandlecache -lumiera_filehandlecache_destroy (LumieraFilehandlecache self) + +void +lumiera_filehandlecache_delete (void) { - self->available += self->cache.cached; - lumiera_mrucache_destroy (&self->cache); - lumiera_mutex_destroy (&self->lock); - return self; + if (fhcache) + { + REQUIRE (!fhcache->checked_out, "Filehandles in use at shutdown"); + RESOURCE_FORGET (filehandlecache, fhcache->rh); + lumiera_mrucache_destroy (&fhcache->cache); + lumiera_mutex_destroy (&fhcache->lock); + free (fhcache); + fhcache = NULL; + } } LumieraFilehandle -lumiera_filehandlecache_filehandle_acquire (LumieraFilehandlecache self) +lumiera_filehandlecache_handle_acquire (LumieraFilehandlecache self, LumieraFiledescriptor desc) { - LumieraFilehandle ret; - LUMIERA_MUTEX_SECTION (&self->lock) + TRACE (filehandlecache); + LumieraFilehandle ret = NULL; + LUMIERA_MUTEX_SECTION (filehandlecache, self->rh, &self->lock) { if (self->available <= 0 && self->cache.cached) { @@ -65,19 +84,58 @@ lumiera_filehandlecache_filehandle_acquire (LumieraFilehandlecache self) } else { - /* allocate new filehandle if we are below the limit or no cached handles where available (overallocating) */ + /* allocate new filehandle if we are below the limit or no cached handles are available (overallocating) */ ret = lumiera_filehandle_new (); if (!ret) - LUMIERA_ERROR_SET (lumiera_filehandlecache, FILEHANDLECACHE_NOHANDLE); + LUMIERA_ERROR_SET (filehandlecache, FILEHANDLECACHE_NOHANDLE); else --self->available; } + ret->use_cnt = 1; + ret->descriptor = desc; + desc->handle = ret; + ++self->checked_out; } return ret; } -void -lumiera_filehandlecache_add (LumieraFilehandlecache self, LumieraFilehandle handle) + +LumieraFilehandle +lumiera_filehandlecache_checkout (LumieraFilehandlecache self, LumieraFilehandle handle) { - UNIMPLEMENTED (""); + REQUIRE (self); + REQUIRE (handle); + /* This function is called with the associated descriptor locked, nothing can modify 'handle' */ + if (!handle->use_cnt) + { + /* lock cache and checkout */ + LUMIERA_MUTEX_SECTION (filehandlecache, self->rh, &self->lock) + { + lumiera_mrucache_checkout (&self->cache, &handle->cachenode); + } + } + ++handle->use_cnt; + ++self->checked_out; + + return handle; } + +void +lumiera_filehandlecache_checkin (LumieraFilehandlecache self, LumieraFilehandle handle) +{ + REQUIRE (self); + REQUIRE (handle); + REQUIRE (handle->use_cnt); + + /* This function is called with the associated descriptor locked, nothing can modify 'self' */ + if (!--handle->use_cnt) + { + /* lock cache and checin */ + LUMIERA_MUTEX_SECTION (filehandlecache, self->rh, &self->lock) + { + --self->checked_out; + lumiera_mrucache_checkin (&self->cache, &handle->cachenode); + } + } +} + diff --git a/src/backend/filehandlecache.h b/src/backend/filehandlecache.h index 40386defe..8d8de44cb 100644 --- a/src/backend/filehandlecache.h +++ b/src/backend/filehandlecache.h @@ -25,6 +25,9 @@ #include "lib/mrucache.h" #include "lib/mutex.h" +typedef struct lumiera_filehandlecache_struct lumiera_filehandlecache; +typedef lumiera_filehandlecache* LumieraFilehandlecache; + #include "backend/filehandle.h" /** @@ -43,55 +46,55 @@ struct lumiera_filehandlecache_struct { lumiera_mrucache cache; int available; + int checked_out; lumiera_mutex lock; + RESOURCE_HANDLE (rh); }; -typedef struct lumiera_filehandlecache_struct lumiera_filehandlecache; -typedef lumiera_filehandlecache* LumieraFilehandlecache; +extern LumieraFilehandlecache fhcache; /** - * Initializes a filehandle cache. - * @param self pointer to the cache to be initialized + * Initializes the filehandle cache. * @param max_entries number how much filehandles shall be managed - * @return self as given * The number of elements the cache can hold is static and should be - * determined by sysconf (_SC_OPEN_MAX) minus some safety margin. + * determined by sysconf (_SC_OPEN_MAX) minus some (big) safety margin. */ -LumieraFilehandlecache -lumiera_filehandlecache_init (LumieraFilehandlecache self, int max_entries); +void +lumiera_filehandlecache_new (int max_entries); /** - * Destroy a filehandle cache. - * @param self the cache to be destroyed - * @return self as given to the now uninitialized! cache + * Delete the filehandle cache. * No filehandles in the cache must be locked, this would be a fatal error. * The handles are closed automatically. */ -LumieraFilehandlecache -lumiera_filehandlecache_destroy (LumieraFilehandlecache self); +void +lumiera_filehandlecache_delete (void); /** * Get a fresh filehandle. * @param self pointer to the cache * @return the new filehandle - * one must call lumiera_mutexacquirer_unlock (&lock) to free the lock on the handle when done */ LumieraFilehandle -lumiera_filehandlecache_filehandle_acquire (LumieraFilehandlecache self); +lumiera_filehandlecache_handle_acquire (LumieraFilehandlecache self, LumieraFiledescriptor desc); /** - * Add filehande back to cache. + * Add filehande back to cache, the filehandle becomes subject of aging. * @param self pointer to the cache * @param handle filehandle to be put back */ void -lumiera_filehandlecache_add (LumieraFilehandlecache self, LumieraFilehandle handle); +lumiera_filehandlecache_add_filehandle (LumieraFilehandlecache self, LumieraFilehandle handle); /** - * + * Remove a filehandle from cache aging + * Filehandles which are subject of cache aging must be checked out before they can be used + * */ LumieraFilehandle -lumiera_filehandlecache_remove (LumieraFilehandlecache self, int fd); +lumiera_filehandlecache_checkout (LumieraFilehandlecache self, LumieraFilehandle handle); +void +lumiera_filehandlecache_checkin (LumieraFilehandlecache self, LumieraFilehandle handle); #endif diff --git a/tests/20filehandle.tests b/tests/20filehandle.tests new file mode 100644 index 000000000..068304f54 --- /dev/null +++ b/tests/20filehandle.tests @@ -0,0 +1,7 @@ + +TESTING "Filehandle management" ./test-filehandles + + +TEST "basic filehandle usage" basic < + + This program is free software; you can redistribute it and/or + modify it under the terms of the GNU General Public License as + published by the Free Software Foundation; either version 2 of the + License, or (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, write to the Free Software + Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA. +*/ +#define _GNU_SOURCE + +#include "lib/llist.h" + +#include "backend/backend.h" +#include "backend/filehandlecache.h" + +#include "tests/test.h" + +TESTS_BEGIN + +TEST ("basic") +{ + lumiera_backend_init (); + LumieraFile file = lumiera_file_new (",tmp_testfile", LUMIERA_FILE_CREATE); + + /* get the filehandle */ + int fd = lumiera_file_handle_acquire (file); + + /* we now 'own'it and can use it */ + ENSURE (fd > -1); + printf ("got filehandle #%d\n", fd); + + /* put it into aging, can't use anymore */ + lumiera_file_handle_release (file); + + lumiera_file_delete (file); + lumiera_backend_destroy (); +} + + +TEST ("more") +{ + lumiera_backend_init (); + lumiera_backend_destroy (); +} + +TESTS_END From dd63a0e1902a8a49f3b730b3729046e42097abdd Mon Sep 17 00:00:00 2001 From: Christian Thaeter Date: Tue, 8 Apr 2008 07:46:25 +0200 Subject: [PATCH 30/39] filedescriptor cleanup and fixes basically working now, flag masking is not complete yet. --- src/backend/filedescriptor.c | 58 ++++++++++++++------------------ tests/20filehandle.tests | 6 ++++ tests/backend/test-filehandles.c | 31 +++++++++++++++++ 3 files changed, 63 insertions(+), 32 deletions(-) diff --git a/src/backend/filedescriptor.c b/src/backend/filedescriptor.c index be166d9c7..0df739670 100644 --- a/src/backend/filedescriptor.c +++ b/src/backend/filedescriptor.c @@ -110,54 +110,48 @@ lumiera_filedescriptor_registry_destroy (void) LumieraFiledescriptor lumiera_filedescriptor_acquire (const char* name, int flags) { - TRACE (filedescriptor); + TRACE (filedescriptor, "%s", name); REQUIRE (registry, "not initialized"); - UNCHECKED; + lumiera_mutexacquirer registry_lock; lumiera_mutexacquirer_init_mutex (®istry_lock, ®istry_mutex, LUMIERA_LOCKED); lumiera_filedescriptor fdesc; fdesc.flags = flags&LUMIERA_FILE_MASK; - for (int retry = 0; !retry; ++retry) + if (stat (name, &fdesc.stat) != 0) { - if (stat (name, &fdesc.stat) == 0) - break; - else + if (errno == ENOENT && flags&O_CREAT) { - if (errno == ENOENT && flags&O_CREAT) + char* dir = lumiera_tmpbuf_strndup (name, PATH_MAX); + char* slash = dir; + while ((slash = strchr (slash+1, '/'))) { - INFO (filedescriptor, "try creating file: %s %d", name, PATH_MAX); - char* dir = lumiera_tmpbuf_strndup (name, PATH_MAX); - char* slash = dir; - - while ((slash = strchr (slash+1, '/'))) - { - *slash = '\0'; - INFO (filedescriptor, "try creating dir: %s", dir); - if (mkdir (dir, 0777) == -1) - { - LUMIERA_ERROR_SET (filedescriptor, ERRNO); - goto efile; - } - *slash = '/'; - } - - int fd; - INFO (filedescriptor, "try creating file: %s", name); - fd = creat (name, 0777); - if (fd == -1) + *slash = '\0'; + INFO (filedescriptor, "try creating dir: %s", dir); + if (mkdir (dir, 0777) == -1 && errno != EEXIST) { LUMIERA_ERROR_SET (filedescriptor, ERRNO); goto efile; } - close (fd); - continue; /* will now retry the fstat once */ + *slash = '/'; + } + int fd; + INFO (filedescriptor, "try creating file: %s", name); + fd = creat (name, 0777); + if (fd == -1) + { + LUMIERA_ERROR_SET (filedescriptor, ERRNO); + goto efile; + } + close (fd); + if (stat (name, &fdesc.stat) != 0) + { + /* finally, no luck */ + LUMIERA_ERROR_SET (filedescriptor, ERRNO); + goto efile; } } - /* finally, no luck */ - LUMIERA_ERROR_SET (filedescriptor, ERRNO); - goto efile; } /* lookup/create descriptor */ diff --git a/tests/20filehandle.tests b/tests/20filehandle.tests index 068304f54..0365bbffc 100644 --- a/tests/20filehandle.tests +++ b/tests/20filehandle.tests @@ -5,3 +5,9 @@ TESTING "Filehandle management" ./test-filehandles TEST "basic filehandle usage" basic < -1); + printf ("got filehandle #%d\n", fds[i]); + } + + /* put them into aging, can't use anymore */ + for (int i=0; i<100; ++i) + { + lumiera_file_handle_release (files[i]); + } + + /* cleanup */ + for (int i=0; i<100; ++i) + { + lumiera_file_delete (files[i]); + } + lumiera_backend_destroy (); } From 8311316a9210ff719c1c061d5899c07cfff76447 Mon Sep 17 00:00:00 2001 From: Christian Thaeter Date: Wed, 9 Apr 2008 07:56:39 +0200 Subject: [PATCH 31/39] Doxydocing for the filehandling stuff --- src/backend/file.h | 34 +++++++++++++++++++++++++----- src/backend/filedescriptor.c | 25 +++++++--------------- src/backend/filedescriptor.h | 39 +++++++++++++++++++++++++++++++++++ src/backend/filehandle.h | 15 ++++++++++++-- src/backend/filehandlecache.h | 13 +++++++++--- 5 files changed, 98 insertions(+), 28 deletions(-) diff --git a/src/backend/file.h b/src/backend/file.h index 60c35447c..9e774b94d 100644 --- a/src/backend/file.h +++ b/src/backend/file.h @@ -63,33 +63,57 @@ struct lumiera_file_struct }; /** - * + * Initialize a file structure. + * @param self pointer to the file structure + * @param name filename + * @param flags open flags + * @return self */ LumieraFile lumiera_file_init (LumieraFile self, const char* name, int flags); /** - * + * Destroy a file structure. + * frees all associated resources, releases the filedescriptor etc. + * @param self file structure to be destroyed + * @return self */ LumieraFile lumiera_file_destroy (LumieraFile self); /** - * + * Allocate a new file structure. + * @param name filename + * @param flags open flags + * @return new file structure */ LumieraFile lumiera_file_new (const char* name, int flags); /** - * + * Frees a file structure. + * @param self file structure to be freed */ void lumiera_file_delete (LumieraFile self); - +/** + * Get a POSIX filehandle for a file. + * Filehandles are opened on demand and must be acquired for use. + * Using filehandles is refcounted and might be nested. + * After using them they must be released which puts them back into filehandlecache aging. + * @param self file structure + * @return POSIX filehandle or -1 on error, check lumiera_error() to retrieve the errorcode + * Currently only LUMIERA_ERROR_ERRNO will be raised but this might change in future. + * Opening files can fail for many reasons and at any time! + */ int lumiera_file_handle_acquire (LumieraFile self); +/** + * Put filehandle back into cache aging. + * @param self file which handle to be released + */ void lumiera_file_handle_release (LumieraFile self); diff --git a/src/backend/filedescriptor.c b/src/backend/filedescriptor.c index 0df739670..5a513a7b4 100644 --- a/src/backend/filedescriptor.c +++ b/src/backend/filedescriptor.c @@ -82,6 +82,7 @@ lumiera_filedescriptor_registry_init (void) { NOBUG_INIT_FLAG (filedescriptor); TRACE (filedescriptor); + REQUIRE (!registry); registry = cuckoo_new (h1, h2, h3, cmp, sizeof (LumieraFiledescriptor), @@ -95,18 +96,12 @@ lumiera_filedescriptor_registry_destroy (void) { TRACE (filedescriptor); REQUIRE (!cuckoo_nelements (registry)); - cuckoo_free (registry); + if (registry) + cuckoo_free (registry); + registry = NULL; } -/** - * Find existing filedescriptor or create one - * @param descriptor strong reference to be initialized with the filedescriptor, - * a filedescriptor will be deleted when its last string reference gets deleted. - * @param name name of the file - * @param flags opening flags for the filedescriptor - * @return descriptor on success or NULL on error - */ LumieraFiledescriptor lumiera_filedescriptor_acquire (const char* name, int flags) { @@ -184,6 +179,7 @@ lumiera_filedescriptor_acquire (const char* name, int flags) return NULL; } + void lumiera_filedescriptor_release (LumieraFiledescriptor self) { @@ -192,11 +188,7 @@ lumiera_filedescriptor_release (LumieraFiledescriptor self) lumiera_filedescriptor_delete (self); } -/** - * Allocate a new filedescriptor cloned from a template - * @param template the source filedescriptor - * @return the constrccted filedescriptor - */ + LumieraFiledescriptor lumiera_filedescriptor_new (LumieraFiledescriptor template) { @@ -219,10 +211,7 @@ lumiera_filedescriptor_new (LumieraFiledescriptor template) return self; } -/** - * delete a filedescriptor and free its resources - * @param descriptor filedescriptor to be freed - */ + void lumiera_filedescriptor_delete (LumieraFiledescriptor self) { diff --git a/src/backend/filedescriptor.h b/src/backend/filedescriptor.h index 817e8bc13..8ae19a04d 100644 --- a/src/backend/filedescriptor.h +++ b/src/backend/filedescriptor.h @@ -36,6 +36,12 @@ typedef lumiera_filedescriptor* LumieraFiledescriptor; #include "backend/filehandle.h" #include "backend/file.h" +/** + * @file Filedescriptors + * Filedescriptors are the underlying working horse in accessing files. + */ + + struct lumiera_filedescriptor_struct { @@ -52,21 +58,54 @@ struct lumiera_filedescriptor_struct RESOURCE_HANDLE (rh); }; +/** + * Initialize the global filedescriptor registry. + * All filedescriptors are looked up by dev/ino/open-flags in a hashtable. Opening hardlinked + * files will be targeted to the same filedescriptor. + * This function never fails but dies on error. + */ void lumiera_filedescriptor_registry_init (void); +/** + * Destroy and free the global filedescriptor registry. + * Never fails. + */ void lumiera_filedescriptor_registry_destroy (void); + +/** + * Find existing filedescriptor or create one + * @param name name of the file + * @param flags opening flags for the filedescriptor + * @return descriptor on success or NULL on error + */ LumieraFiledescriptor lumiera_filedescriptor_acquire (const char* name, int flags); + +/** + * Release a filedescriptor. + * @param self filedescriptor to be released + */ void lumiera_filedescriptor_release (LumieraFiledescriptor self); + +/** + * Allocate a new filedescriptor cloned from a template + * @param template the source filedescriptor + * @return the constrccted filedescriptor + */ LumieraFiledescriptor lumiera_filedescriptor_new (LumieraFiledescriptor template); +/** + * Delete a filedescriptor + * Called whenever its reference count drops to zero. + * @param self the filedescriptor to be deleted + */ void lumiera_filedescriptor_delete (LumieraFiledescriptor self); diff --git a/src/backend/filehandle.h b/src/backend/filehandle.h index 54e4d6936..47fd0f3a7 100644 --- a/src/backend/filehandle.h +++ b/src/backend/filehandle.h @@ -31,6 +31,9 @@ typedef lumiera_filehandle* LumieraFilehandle; /** * @file Filehandles + * Filehandles manage the underlying POSIX filehandle for a filedescriptor. + * Since we want to support handling of more files than POSIX filehandles are available on a common system + * the filehandles are opened, cached and closed on demand, see 'filehandlecache'. */ @@ -45,12 +48,20 @@ struct lumiera_filehandle_struct LumieraFiledescriptor descriptor; }; +/** + * Allocate a new filehandle structure. + * @return new filehandle structure + */ LumieraFilehandle lumiera_filehandle_new (); -int -lumiera_filehandle_get (LumieraFilehandle self); +/** + * destroy the resources associated eith a filehandle structure. + * This function is used by the filehandlecache to recycle filehandle structs. + * @param node pointer to the cachenode member of a struct filehandle + * @return pointer to the start of the memory of the destroyed filehandle + */ void* lumiera_filehandle_destroy_node (LList node); diff --git a/src/backend/filehandlecache.h b/src/backend/filehandlecache.h index 8d8de44cb..fd6ddfd13 100644 --- a/src/backend/filehandlecache.h +++ b/src/backend/filehandlecache.h @@ -35,10 +35,10 @@ typedef lumiera_filehandlecache* LumieraFilehandlecache; * The number of filehandles a program can held open is usually limited, since we want to support * using a less limited number of files and closing/opening for each operation is expensive, we * provide a cache to keep the most frequent used files open and gracefully close/recycle unused filehandles. + * The filehandlecache defined here protects all operations on the cache with a mutex. */ - /* File handle cache manages file handles */ @@ -88,12 +88,19 @@ lumiera_filehandlecache_add_filehandle (LumieraFilehandlecache self, LumieraFile /** * Remove a filehandle from cache aging - * Filehandles which are subject of cache aging must be checked out before they can be used - * + * Filehandles which are subject of cache aging must be checked out before they can be used. + * @param self the filehandlecache + * @param handle the filehandle to be checked out */ LumieraFilehandle lumiera_filehandlecache_checkout (LumieraFilehandlecache self, LumieraFilehandle handle); +/** + * Put a filehandle into the cache + * Filehandles which are checked in are subject of cache aging and might get destroyed and reused. + * @param self the filehandlecache + * @param handle the filehandle to be checked in + */ void lumiera_filehandlecache_checkin (LumieraFilehandlecache self, LumieraFilehandle handle); From 4ed2fe9e7cf4503b4f94f5b9b2368c07b52acbee Mon Sep 17 00:00:00 2001 From: Christian Thaeter Date: Wed, 9 Apr 2008 07:58:29 +0200 Subject: [PATCH 32/39] few more todos in the wiki, plan for a player in roadmap --- wiki/todo.html | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/wiki/todo.html b/wiki/todo.html index e66ec929f..763c8a80a 100644 --- a/wiki/todo.html +++ b/wiki/todo.html @@ -3350,6 +3350,11 @@ The idea here is to show and measure backend and scheduler performance backend to be done. +
+
Use the backend, gmerlin/avdecoder and a Player widget to playback videos. This should be a small application which can open and playback many videos from on instance in parallel. Later on scrubbing for this videos might be supported.
+
+The idea here is to show and measure backend and scheduler performance and beeing a proof of concept for the design. It will run independent of the Proc layer but needs quite some work on the backend to be done.
+
From 00b9dff106d018c03a704e3b5db1e92467c25f3b Mon Sep 17 00:00:00 2001 From: Christian Thaeter Date: Thu, 10 Apr 2008 02:40:34 +0200 Subject: [PATCH 33/39] Doxygen config update * set EXPORT_ALL=NO to leave out undcoumented entities. * Fix a lot @file directives --- src/backend/file.h | 3 ++- src/backend/filedescriptor.h | 4 +++- src/backend/filehandle.h | 3 ++- src/backend/filehandlecache.h | 3 ++- src/lib/condition.c | 3 ++- src/lib/condition.h | 3 ++- src/lib/cuckoo.h | 3 ++- src/lib/error.c | 3 ++- src/lib/error.h | 3 ++- src/lib/llist.h | 3 ++- src/lib/locking.h | 3 ++- src/lib/mrucache.h | 3 ++- src/lib/mutex.c | 3 ++- src/lib/mutex.h | 3 ++- src/lib/plugin.c | 3 ++- src/lib/plugin.h | 3 ++- src/lib/references.c | 3 ++- src/lib/references.h | 3 ++- src/lib/rwlock.c | 3 ++- src/lib/rwlock.h | 3 ++- src/lib/safeclib.c | 3 ++- src/lib/safeclib.h | 3 ++- 22 files changed, 45 insertions(+), 22 deletions(-) diff --git a/src/backend/file.h b/src/backend/file.h index 9e774b94d..175b93e9d 100644 --- a/src/backend/file.h +++ b/src/backend/file.h @@ -30,7 +30,8 @@ NOBUG_DECLARE_FLAG (file); /** - * @file File management + * @file + * File management * Handling Files is splitted into different classes: * 1. The 'lumiera_file' class which acts as interface to the outside for managing files. * 'lumiera_file' is addressed by the name of the file. Since files can have more than one name (hardlinks) diff --git a/src/backend/filedescriptor.h b/src/backend/filedescriptor.h index 8ae19a04d..4789f9ce2 100644 --- a/src/backend/filedescriptor.h +++ b/src/backend/filedescriptor.h @@ -37,8 +37,10 @@ typedef lumiera_filedescriptor* LumieraFiledescriptor; #include "backend/file.h" /** - * @file Filedescriptors + * @file + * Filedescriptors. * Filedescriptors are the underlying working horse in accessing files. + * All information associated with managing a file is kept here. */ diff --git a/src/backend/filehandle.h b/src/backend/filehandle.h index 47fd0f3a7..e1b02b2e4 100644 --- a/src/backend/filehandle.h +++ b/src/backend/filehandle.h @@ -30,7 +30,8 @@ typedef lumiera_filehandle* LumieraFilehandle; #include "backend/filedescriptor.h" /** - * @file Filehandles + * @file + * Filehandles. * Filehandles manage the underlying POSIX filehandle for a filedescriptor. * Since we want to support handling of more files than POSIX filehandles are available on a common system * the filehandles are opened, cached and closed on demand, see 'filehandlecache'. diff --git a/src/backend/filehandlecache.h b/src/backend/filehandlecache.h index fd6ddfd13..25964f986 100644 --- a/src/backend/filehandlecache.h +++ b/src/backend/filehandlecache.h @@ -31,7 +31,8 @@ typedef lumiera_filehandlecache* LumieraFilehandlecache; #include "backend/filehandle.h" /** - * @file Filehandle management and caching + * @file + * Filehandle management and caching * The number of filehandles a program can held open is usually limited, since we want to support * using a less limited number of files and closing/opening for each operation is expensive, we * provide a cache to keep the most frequent used files open and gracefully close/recycle unused filehandles. diff --git a/src/lib/condition.c b/src/lib/condition.c index 1bfa2ef96..60e8b6f10 100644 --- a/src/lib/condition.c +++ b/src/lib/condition.c @@ -22,7 +22,8 @@ #include "lib/condition.h" /** - * @file Condition variables + * @file + * Condition variables */ LUMIERA_ERROR_DEFINE (CONDITION_DESTROY, "condition destroy failed"); diff --git a/src/lib/condition.h b/src/lib/condition.h index ffc00523c..dd92f326b 100644 --- a/src/lib/condition.h +++ b/src/lib/condition.h @@ -25,7 +25,8 @@ #include "lib/locking.h" /** - * @file Condition variables, header + * @file + * Condition variables, header */ LUMIERA_ERROR_DECLARE (CONDITION_DESTROY); diff --git a/src/lib/cuckoo.h b/src/lib/cuckoo.h index 63b6d260d..323b5cc75 100644 --- a/src/lib/cuckoo.h +++ b/src/lib/cuckoo.h @@ -22,7 +22,8 @@ #define CUCKOO_H /** - * @file Cuckoo hashing + * @file + * Cuckoo hashing * This hashing gives guaranteed O(1) lookup complexity and amortized O(1) insert and remove complexity. * Hash tables by default grow and shrink automatically. It is posible to preallocate entries and turn * automatic shrinking off taking out the memory management factors for insert and remove operations. diff --git a/src/lib/error.c b/src/lib/error.c index 81e24bad8..b2bc390b6 100644 --- a/src/lib/error.c +++ b/src/lib/error.c @@ -24,7 +24,8 @@ #include "lib/error.h" /** - * @file C Error handling in Lumiera. + * @file + * C Error handling in Lumiera. */ diff --git a/src/lib/error.h b/src/lib/error.h index e71c0cdfa..1eeb42dda 100644 --- a/src/lib/error.h +++ b/src/lib/error.h @@ -31,7 +31,8 @@ extern "C" { #include /** - * @file C Error handling in Lumiera, header. + * @file + * C Error handling in Lumiera, header. */ diff --git a/src/lib/llist.h b/src/lib/llist.h index efb3f5202..128a3e826 100644 --- a/src/lib/llist.h +++ b/src/lib/llist.h @@ -27,7 +27,8 @@ #include /** - * @file Intrusive cyclic double linked list + * @file + * Intrusive cyclic double linked list * There is only one node type which contains a forward and a backward pointer. In a empty initialized node, * this pointers point to the node itself. Note that these pointers can never ever become NULL. * This lists are used by using one node as 'root' node where its both pointers are the head/tail pointer to the actual list. diff --git a/src/lib/locking.h b/src/lib/locking.h index f62dd6c29..3c4d497d8 100644 --- a/src/lib/locking.h +++ b/src/lib/locking.h @@ -34,7 +34,8 @@ LUMIERA_ERROR_DECLARE (MUTEX_UNLOCK); LUMIERA_ERROR_DECLARE (MUTEX_DESTROY); /** - * @file Shared declarations for all locking primitives. + * @file + * Shared declarations for all locking primitives. */ /** diff --git a/src/lib/mrucache.h b/src/lib/mrucache.h index c81bd7599..08654754f 100644 --- a/src/lib/mrucache.h +++ b/src/lib/mrucache.h @@ -27,7 +27,8 @@ #include /** - * @file Most recent used cache + * @file + * Most recent used cache * Elements (addressed by a LList node) are either checked in the cache and thereby subject of aging * or checked out under control of the user. Most operations require that the chache is locked. This locking * must be done from elsewhere. diff --git a/src/lib/mutex.c b/src/lib/mutex.c index 009502e5a..b7c5f1844 100644 --- a/src/lib/mutex.c +++ b/src/lib/mutex.c @@ -22,7 +22,8 @@ #include "lib/mutex.h" /** - * @file Mutual exclusion locking. + * @file + * Mutual exclusion locking. */ LUMIERA_ERROR_DEFINE (MUTEX_LOCK, "Mutex locking failed"); diff --git a/src/lib/mutex.h b/src/lib/mutex.h index ca48ea592..f19058e7a 100644 --- a/src/lib/mutex.h +++ b/src/lib/mutex.h @@ -25,7 +25,8 @@ #include "lib/locking.h" /** - * @file Mutual exclusion locking, header. + * @file + * Mutual exclusion locking, header. */ #define LUMIERA_MUTEX_SECTION(flag, handle, mutex) \ RESOURCE_HANDLE (rh_##__LINE__##_); \ diff --git a/src/lib/plugin.c b/src/lib/plugin.c index 8345d9887..1f5e8576e 100644 --- a/src/lib/plugin.c +++ b/src/lib/plugin.c @@ -31,7 +31,8 @@ #include "safeclib.h" /** - * @file Plugin loader. + * @file + * Plugin loader. */ diff --git a/src/lib/plugin.h b/src/lib/plugin.h index f8916ae96..54dad6372 100644 --- a/src/lib/plugin.h +++ b/src/lib/plugin.h @@ -33,7 +33,8 @@ extern "C" { #include "error.h" /** - * @file Plugin loader, header. + * @file + * Plugin loader, header. */ diff --git a/src/lib/references.c b/src/lib/references.c index 3e9d5f260..925258377 100644 --- a/src/lib/references.c +++ b/src/lib/references.c @@ -22,7 +22,8 @@ #include "lib/safeclib.h" /** - * @file Strong and Weak references + * @file + * Strong and Weak references * Strong references keep some object alive while they existing * Weak references become invalidated when the referenced object gets destroyed * diff --git a/src/lib/references.h b/src/lib/references.h index 3e00457e7..6ad65d62f 100644 --- a/src/lib/references.h +++ b/src/lib/references.h @@ -25,7 +25,8 @@ #include /** - * @file Strong and Weak references, header. + * @file + * Strong and Weak references, header. */ diff --git a/src/lib/rwlock.c b/src/lib/rwlock.c index 092c7b54b..077ff9766 100644 --- a/src/lib/rwlock.c +++ b/src/lib/rwlock.c @@ -29,7 +29,8 @@ LUMIERA_ERROR_DEFINE(RWLOCK_RLOCK, "rlock"); LUMIERA_ERROR_DEFINE(RWLOCK_WLOCK, "wlock"); /** - * @file Read/write locks. + * @file + * Read/write locks. */ diff --git a/src/lib/rwlock.h b/src/lib/rwlock.h index 5e121fbcc..d58627dda 100644 --- a/src/lib/rwlock.h +++ b/src/lib/rwlock.h @@ -39,7 +39,8 @@ LUMIERA_ERROR_DECLARE(RWLOCK_RLOCK); LUMIERA_ERROR_DECLARE(RWLOCK_WLOCK); /** - * @file Read/write locks, header. + * @file + * Read/write locks, header. */ diff --git a/src/lib/safeclib.c b/src/lib/safeclib.c index 0e82e2dc0..f500ea71c 100644 --- a/src/lib/safeclib.c +++ b/src/lib/safeclib.c @@ -27,7 +27,8 @@ #include /** - * @file Portable and safe wrapers around some clib functions and some tools + * @file + * Portable and safe wrapers around some clib functions and some tools */ LUMIERA_ERROR_DEFINE (NO_MEMORY, "Out of Memory!"); diff --git a/src/lib/safeclib.h b/src/lib/safeclib.h index f486ed056..6fc532239 100644 --- a/src/lib/safeclib.h +++ b/src/lib/safeclib.h @@ -24,7 +24,8 @@ #include /** - * @file Portable and safe wrapers around some clib functions and some tools + * @file + * Portable and safe wrapers around some clib functions and some tools */ LUMIERA_ERROR_DECLARE(NO_MEMORY); From c4816c39fdc802a4db00e65dfab12077e18970c3 Mon Sep 17 00:00:00 2001 From: Christian Thaeter Date: Thu, 10 Apr 2008 06:03:21 +0200 Subject: [PATCH 34/39] Proper flag and stat handling for files Filedescriptors create files at construction, each later access becomes a reopen on the now existing file. file_handle_acquire() reopens the file to gain a handle when required and checks that the file is really the one we expected (ino and dev num) --- src/backend/file.c | 29 ++++++++++++++++++++++++++--- src/backend/file.h | 3 ++- 2 files changed, 28 insertions(+), 4 deletions(-) diff --git a/src/backend/file.c b/src/backend/file.c index e9cd208dc..861f3ce21 100644 --- a/src/backend/file.c +++ b/src/backend/file.c @@ -27,9 +27,13 @@ #include "backend/filedescriptor.h" #include +#include NOBUG_DEFINE_FLAG_PARENT (file, file_all); +LUMIERA_ERROR_DEFINE(FILE_CHANGED, "File changed unexpected"); + + LumieraFile lumiera_file_init (LumieraFile self, const char* name, int flags) { @@ -85,12 +89,31 @@ lumiera_file_handle_acquire (LumieraFile self) if (self->descriptor->handle->fd == -1) { - TODO ("stat handling/update"); - self->descriptor->handle->fd = open (self->name, self->descriptor->flags); - if (self->descriptor->handle->fd == -1) + int fd; + fd = open (self->name, self->descriptor->flags & ~(O_EXCL|O_TRUNC|O_CREAT)); + if (fd == -1) { LUMIERA_ERROR_SET (file, ERRNO); + return -1; } + else + { + struct stat st; + if (fstat (fd, &st) == -1) + { + close (fd); + LUMIERA_ERROR_SET (file, ERRNO); + return -1; + } + else if (self->descriptor->stat.st_dev != st.st_dev || self->descriptor->stat.st_ino != st.st_ino) + { + close (fd); + /* Woops this is not the file we expected to use */ + LUMIERA_ERROR_SET (file, FILE_CHANGED); + return -1; + } + } + self->descriptor->handle->fd = fd; } } diff --git a/src/backend/file.h b/src/backend/file.h index 175b93e9d..71ca15751 100644 --- a/src/backend/file.h +++ b/src/backend/file.h @@ -29,6 +29,8 @@ NOBUG_DECLARE_FLAG (file); +LUMIERA_ERROR_DECLARE(FILE_CHANGED); + /** * @file * File management @@ -48,7 +50,6 @@ typedef lumiera_file* LumieraFile; #include "backend/filehandle.h" #include "backend/filedescriptor.h" - #define LUMIERA_FILE_READONLY (O_RDONLY | O_LARGEFILE | O_NOATIME) #define LUMIERA_FILE_READWRITE (O_RDWR | O_LARGEFILE | O_NOATIME) #define LUMIERA_FILE_CREATE (O_RDWR | O_LARGEFILE | O_NOATIME | O_CREAT | O_EXCL) From 444b142d854ed51c727db86dd2fd280189e107df Mon Sep 17 00:00:00 2001 From: Christian Thaeter Date: Thu, 10 Apr 2008 06:07:04 +0200 Subject: [PATCH 35/39] Correct File flag masking The hash lookup needs to ignore O_EXCL|O_TRUNC|O_CREAT as well as file reopening must not use this flags. --- src/backend/file.c | 2 +- src/backend/file.h | 2 +- src/backend/filedescriptor.c | 15 +++++++++------ src/backend/filedescriptor.h | 1 - 4 files changed, 11 insertions(+), 9 deletions(-) diff --git a/src/backend/file.c b/src/backend/file.c index 861f3ce21..e315537fc 100644 --- a/src/backend/file.c +++ b/src/backend/file.c @@ -90,7 +90,7 @@ lumiera_file_handle_acquire (LumieraFile self) if (self->descriptor->handle->fd == -1) { int fd; - fd = open (self->name, self->descriptor->flags & ~(O_EXCL|O_TRUNC|O_CREAT)); + fd = open (self->name, self->descriptor->flags & LUMIERA_FILE_MASK); if (fd == -1) { LUMIERA_ERROR_SET (file, ERRNO); diff --git a/src/backend/file.h b/src/backend/file.h index 71ca15751..bba12a167 100644 --- a/src/backend/file.h +++ b/src/backend/file.h @@ -56,7 +56,7 @@ typedef lumiera_file* LumieraFile; #define LUMIERA_FILE_RECREATE (O_RDWR | O_LARGEFILE | O_NOATIME | O_CREAT | O_TRUNC) /* creat and excl flags will be masked out for descriptor lookup */ -#define LUMIERA_FILE_MASK ~(O_CREAT | O_EXCL) +#define LUMIERA_FILE_MASK ~(O_CREAT | O_EXCL | O_TRUNC) struct lumiera_file_struct { diff --git a/src/backend/filedescriptor.c b/src/backend/filedescriptor.c index 5a513a7b4..b9608984f 100644 --- a/src/backend/filedescriptor.c +++ b/src/backend/filedescriptor.c @@ -51,21 +51,24 @@ static size_t h1 (const void* item, const uint32_t r) { const LumieraFiledescriptor i = *(const LumieraFiledescriptor*)item; - return i->stat.st_dev^i->stat.st_ino^i->flags^((i->stat.st_dev^i->stat.st_ino^i->flags)>>7)^r; + return i->stat.st_dev^i->stat.st_ino^(i->flags&LUMIERA_FILE_MASK) + ^((i->stat.st_dev^i->stat.st_ino^(i->flags&LUMIERA_FILE_MASK))>>7)^r; } static size_t h2 (const void* item, const uint32_t r) { const LumieraFiledescriptor i = *(const LumieraFiledescriptor*)item; - return i->stat.st_dev^i->stat.st_ino^i->flags^((i->stat.st_dev^i->stat.st_ino^i->flags)>>5)^r; + return i->stat.st_dev^i->stat.st_ino^(i->flags&LUMIERA_FILE_MASK) + ^((i->stat.st_dev^i->stat.st_ino^(i->flags&LUMIERA_FILE_MASK))>>5)^r; } static size_t h3 (const void* item, const uint32_t r) { const LumieraFiledescriptor i = *(const LumieraFiledescriptor*)item; - return i->stat.st_dev^i->stat.st_ino^i->flags^((i->stat.st_dev^i->stat.st_ino^i->flags)>>3)^r; + return i->stat.st_dev^i->stat.st_ino^(i->flags&LUMIERA_FILE_MASK) + ^((i->stat.st_dev^i->stat.st_ino^(i->flags&LUMIERA_FILE_MASK))>>3)^r; } @@ -74,7 +77,8 @@ cmp (const void* keya, const void* keyb) { const LumieraFiledescriptor a = *(const LumieraFiledescriptor*)keya; const LumieraFiledescriptor b = *(const LumieraFiledescriptor*)keyb; - return a->stat.st_dev == b->stat.st_dev && a->stat.st_ino == b->stat.st_ino && a->flags == b->flags; + return a->stat.st_dev == b->stat.st_dev && a->stat.st_ino == b->stat.st_ino + && (a->flags&LUMIERA_FILE_MASK) == (b->flags&LUMIERA_FILE_MASK); } void @@ -112,7 +116,7 @@ lumiera_filedescriptor_acquire (const char* name, int flags) lumiera_mutexacquirer_init_mutex (®istry_lock, ®istry_mutex, LUMIERA_LOCKED); lumiera_filedescriptor fdesc; - fdesc.flags = flags&LUMIERA_FILE_MASK; + fdesc.flags = flags; if (stat (name, &fdesc.stat) != 0) { @@ -199,7 +203,6 @@ lumiera_filedescriptor_new (LumieraFiledescriptor template) self->flags = template->flags; lumiera_mutex_init (&self->lock); - self->reopened = 0; self->refcount = 1; self->handle = 0; diff --git a/src/backend/filedescriptor.h b/src/backend/filedescriptor.h index 4789f9ce2..528be41e8 100644 --- a/src/backend/filedescriptor.h +++ b/src/backend/filedescriptor.h @@ -50,7 +50,6 @@ struct lumiera_filedescriptor_struct struct stat stat; /* create after first open, maintained metadata, MUST BE FIRST! */ int flags; /* open flags, must be masked for reopen */ lumiera_mutex lock; /* locks operations on this file descriptor */ - unsigned reopened; /* count for reopens 0=not yet opened, 1=first, 2..=reopened */ unsigned refcount; /* reference counter, all users sans registry */ LumieraFilehandle handle; From 241327bcd423fb27a925a0a25b4195b11e1cdae9 Mon Sep 17 00:00:00 2001 From: Christian Thaeter Date: Fri, 2 May 2008 23:55:14 +0200 Subject: [PATCH 36/39] rename 'fhcache' global var to 'lumiera_fhcache' --- src/backend/file.c | 9 +++++---- src/backend/filehandlecache.c | 37 ++++++++++++++++------------------- src/backend/filehandlecache.h | 2 +- 3 files changed, 23 insertions(+), 25 deletions(-) diff --git a/src/backend/file.c b/src/backend/file.c index e315537fc..4a309ed3a 100644 --- a/src/backend/file.c +++ b/src/backend/file.c @@ -77,15 +77,15 @@ lumiera_file_handle_acquire (LumieraFile self) TRACE (file); REQUIRE (self); REQUIRE (self->descriptor); - REQUIRE (fhcache); + REQUIRE (lumiera_fhcache); LUMIERA_MUTEX_SECTION (file, self->descriptor->rh, &self->descriptor->lock) { if (!self->descriptor->handle) /* no handle yet, get a new one */ - lumiera_filehandlecache_handle_acquire (fhcache, self->descriptor); + lumiera_filehandlecache_handle_acquire (lumiera_fhcache, self->descriptor); else - lumiera_filehandlecache_checkout (fhcache, self->descriptor->handle); + lumiera_filehandlecache_checkout (lumiera_fhcache, self->descriptor->handle); if (self->descriptor->handle->fd == -1) { @@ -120,6 +120,7 @@ lumiera_file_handle_acquire (LumieraFile self) return self->descriptor->handle->fd; } + void lumiera_file_handle_release (LumieraFile self) { @@ -127,6 +128,6 @@ lumiera_file_handle_release (LumieraFile self) LUMIERA_MUTEX_SECTION (file, self->descriptor->rh, &self->descriptor->lock) { - lumiera_filehandlecache_checkin (fhcache, self->descriptor->handle); + lumiera_filehandlecache_checkin (lumiera_fhcache, self->descriptor->handle); } } diff --git a/src/backend/filehandlecache.c b/src/backend/filehandlecache.c index c629f765b..181807661 100644 --- a/src/backend/filehandlecache.c +++ b/src/backend/filehandlecache.c @@ -31,38 +31,35 @@ NOBUG_DEFINE_FLAG_PARENT (filehandlecache, file_all); LUMIERA_ERROR_DEFINE (FILEHANDLECACHE_NOHANDLE, "No filehandle available"); -LumieraFilehandlecache fhcache = NULL; +LumieraFilehandlecache lumiera_fhcache = NULL; void lumiera_filehandlecache_new (int max_entries) { - if (!fhcache) - { - NOBUG_INIT_FLAG (filehandlecache); - fhcache = lumiera_malloc (sizeof (lumiera_filehandlecache)); - lumiera_mrucache_init (&fhcache->cache, lumiera_filehandle_destroy_node); - fhcache->available = max_entries; - fhcache->checked_out = 0; - lumiera_mutex_init (&fhcache->lock); - RESOURCE_ANNOUNCE (filehandlecache, "mutex", "filehandlecache", fhcache, fhcache->rh); - } - else - NOTREACHED; + REQUIRE (!lumiera_fhcache, "Filehandlecache already initialized"); + + NOBUG_INIT_FLAG (filehandlecache); + lumiera_fhcache = lumiera_malloc (sizeof (lumiera_filehandlecache)); + lumiera_mrucache_init (&lumiera_fhcache->cache, lumiera_filehandle_destroy_node); + lumiera_fhcache->available = max_entries; + lumiera_fhcache->checked_out = 0; + lumiera_mutex_init (&lumiera_fhcache->lock); + RESOURCE_ANNOUNCE (filehandlecache, "mutex", "filehandlecache", lumiera_fhcache, lumiera_fhcache->rh); } void lumiera_filehandlecache_delete (void) { - if (fhcache) + if (lumiera_fhcache) { - REQUIRE (!fhcache->checked_out, "Filehandles in use at shutdown"); - RESOURCE_FORGET (filehandlecache, fhcache->rh); - lumiera_mrucache_destroy (&fhcache->cache); - lumiera_mutex_destroy (&fhcache->lock); - free (fhcache); - fhcache = NULL; + REQUIRE (!lumiera_fhcache->checked_out, "Filehandles in use at shutdown"); + RESOURCE_FORGET (filehandlecache, lumiera_fhcache->rh); + lumiera_mrucache_destroy (&lumiera_fhcache->cache); + lumiera_mutex_destroy (&lumiera_fhcache->lock); + free (lumiera_fhcache); + lumiera_fhcache = NULL; } } diff --git a/src/backend/filehandlecache.h b/src/backend/filehandlecache.h index 25964f986..37c2a416b 100644 --- a/src/backend/filehandlecache.h +++ b/src/backend/filehandlecache.h @@ -52,7 +52,7 @@ struct lumiera_filehandlecache_struct RESOURCE_HANDLE (rh); }; -extern LumieraFilehandlecache fhcache; +extern LumieraFilehandlecache lumiera_fhcache; /** * Initializes the filehandle cache. From 9ad62d0edc0e7f2bd9ba6f480ec1ea9c18d0f73f Mon Sep 17 00:00:00 2001 From: Christian Thaeter Date: Fri, 2 May 2008 23:56:49 +0200 Subject: [PATCH 37/39] Fix: one must not return from a mutex section --- src/backend/file.c | 3 --- 1 file changed, 3 deletions(-) diff --git a/src/backend/file.c b/src/backend/file.c index 4a309ed3a..9af608567 100644 --- a/src/backend/file.c +++ b/src/backend/file.c @@ -94,7 +94,6 @@ lumiera_file_handle_acquire (LumieraFile self) if (fd == -1) { LUMIERA_ERROR_SET (file, ERRNO); - return -1; } else { @@ -103,14 +102,12 @@ lumiera_file_handle_acquire (LumieraFile self) { close (fd); LUMIERA_ERROR_SET (file, ERRNO); - return -1; } else if (self->descriptor->stat.st_dev != st.st_dev || self->descriptor->stat.st_ino != st.st_ino) { close (fd); /* Woops this is not the file we expected to use */ LUMIERA_ERROR_SET (file, FILE_CHANGED); - return -1; } } self->descriptor->handle->fd = fd; From 64e549dc8125f923aab3d9e02d941f1ddca28dee Mon Sep 17 00:00:00 2001 From: Christian Thaeter Date: Sat, 3 May 2008 16:32:27 +0200 Subject: [PATCH 38/39] headers for the memory mapping backend (WIP) --- src/backend/mmap.h | 88 ++++++++++++++++++++++++++ src/backend/mmapcache.h | 136 ++++++++++++++++++++++++++++++++++++++++ src/backend/mmapings.h | 118 ++++++++++++++++++++++++++++++++++ 3 files changed, 342 insertions(+) create mode 100644 src/backend/mmap.h create mode 100644 src/backend/mmapcache.h create mode 100644 src/backend/mmapings.h diff --git a/src/backend/mmap.h b/src/backend/mmap.h new file mode 100644 index 000000000..608e40133 --- /dev/null +++ b/src/backend/mmap.h @@ -0,0 +1,88 @@ +/* + mmap.h - memory mapped acces to files + + Copyright (C) Lumiera.org + 2008, Christian Thaeter + + This program is free software; you can redistribute it and/or + modify it under the terms of the GNU General Public License as + published by the Free Software Foundation; either version 2 of the + License, or (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, write to the Free Software + Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA. +*/ + +#ifndef LUMIERA_MMAP_H +#define LUMIERA_MMAP_H + +#include "lib/llist.h" + + +typedef struct lumiera_mmap_struct lumiera_mmap; +typedef lumiera_mmap* LumieraMMap; + + +#include "backend/filedescriptor.h" + + +#include +#include + +NOBUG_DECLARE_FLAG (mmap); + +/** + * @file + * MMap objects cover a memory maped range in a file + */ + + + +/** + * mmaped area + * + */ +struct lumiera_mmap_struct +{ + /** used for the mrucache when checked in the cache OR for attaching owners when checked out **/ + llist cachenode; + + /** all mmaps of a file are chained in this list, used to find ranges **/ + llist searchnode; + + off_t start; + size_t size; + void* address; + + /** array with refcounters per page **/ + unsigned short* refmap; + + /** 0 when this mmap is in cache, else the count of attached owners **/ + unsigned use_cnt; + //RESOURCE_HANDLE (rh); +}; + + +LumieraMMap +lumiera_mmap_init (LumieraMMap self, LumieraFile file, LList acquirer, off_t start, size_t size, size_t chunksize); + +LumieraMMap +lumiera_mmap_new (LumieraFile file, LList acquirer, off_t start, size_t size, size_t chunksize); + +void* +lumiera_mmap_destroy_node (LList node); + +#endif +/* +// Local Variables: +// mode: C +// c-file-style: "gnu" +// indent-tabs-mode: nil +// End: +*/ diff --git a/src/backend/mmapcache.h b/src/backend/mmapcache.h new file mode 100644 index 000000000..ed2efcb43 --- /dev/null +++ b/src/backend/mmapcache.h @@ -0,0 +1,136 @@ +/* + mmapcache.h - handle aging of mmap objects + + Copyright (C) Lumiera.org + 2008, Christian Thaeter + + This program is free software; you can redistribute it and/or + modify it under the terms of the GNU General Public License as + published by the Free Software Foundation; either version 2 of the + License, or (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, write to the Free Software + Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA. +*/ + +#ifndef LUMIERA_MMAPCACHE_H +#define LUMIERA_MMAPCACHE_H + +#include "lib/error.h" +#include "lib/mrucache.h" +#include "lib/mutex.h" + + +typedef struct lumiera_mmapcache_struct lumiera_mmapcache; +typedef lumiera_mmapcache* LumieraMMapcache; + +#include "backend/mmap.h" + +#include + +/** + * @file + * Mmapcache stores a MRU cache of all established mmaped memory regions which are currently not in use. + * The mmapcache also manages the upper limit about how much memory can be mmaped. + */ + +struct lumiera_mmapcache_struct +{ + lumiera_mrucache cache; + size_t limit; + size_t total; + size_t cached; + lumiera_mutex lock; + RESOURCE_HANDLE (rh); +}; + +extern LumieraMMapcache lumiera_mcache; + + +/** + * Initializes the mmapcache. + * @param limit the mmapcache will drop elements when the sum of all mmapings gives over limit + */ +void +lumiera_mmapcache_new (size_t limit); + +/** + * Delete the mmap cache. + * No mmaps in the cache must be locked, this would be a fatal error. + * The handles are closed automatically. + */ +void +lumiera_mmapcache_delete (void); + +/** + * Get a fresh mmap object. + * when mmaped_limit is reached, the oldest mmap object gets dropped else a new allocated object + * is returned + * @param self pointer to the cache + * @return the new mmap + */ +LumieraMMap +lumiera_mmapcache_mmap_acquire (LumieraMMapcache self); + + +/** + * Announce a new mmap object to the cache quotas. + * Update the statistics kept in the cache, + * the map object is still considered to be checked out + * @param self pointer to the cache + * @param map object to be announced + */ +void +lumiera_mmapcache_announce (LumieraMMapcache self, LumieraMMap map); + +/** + * Remove a mmap object from the cache quotas. + * Update the statistics kept in the cache, + * called by mmap's destructor only + * @param self pointer to the cache + * @param map object to be removed + */ +void +lumiera_mmapcache_forget (LumieraMMapcache self, LumieraMMap map); + +/** + * Destroy and free the nelem oldest elements. + * Used to free up resources and memory. + * @param self cache where to free elements. + * @return nelem-(numer of elements which got freed), that is 0 if all requested elements got freed + */ +int +lumiera_mmapcache_age (LumieraMMapcache self); + +/** + * Remove a mmap from cache aging + * Mmaps which are subject of cache aging must be checked out before they can be used. + * @param self the mmapcache + * @param handle the mmap to be checked out + */ +LumieraMMap +lumiera_mmapcache_checkout (LumieraMMapcache self, LumieraMMap handle); + +/** + * Put a mmap into the cache + * Mmaps which are checked in are subject of cache aging and might get destroyed and reused. + * @param self the mmapcache + * @param handle the mmap to be checked in + */ +void +lumiera_mmapcache_checkin (LumieraMMapcache self, LumieraMMap handle); + +#endif +/* +// Local Variables: +// mode: C +// c-file-style: "gnu" +// indent-tabs-mode: nil +// End: +*/ diff --git a/src/backend/mmapings.h b/src/backend/mmapings.h new file mode 100644 index 000000000..234ab280b --- /dev/null +++ b/src/backend/mmapings.h @@ -0,0 +1,118 @@ +/* + mmapings.h - manage ranges of mmaped areas on a filedescriptor + + Copyright (C) Lumiera.org + 2008, Christian Thaeter + + This program is free software; you can redistribute it and/or + modify it under the terms of the GNU General Public License as + published by the Free Software Foundation; either version 2 of the + License, or (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, write to the Free Software + Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA. +*/ + +#ifndef LUMIERA_MMAPINGS_H +#define LUMIERA_MMAPINGS_H + +#include "lib/mutex.h" +#include "lib/llist.h" + +typedef struct lumiera_mmapings_struct lumiera_mmapings; +typedef lumiera_mmapings* LumieraMMapings; + +#include "backend/filedescriptor.h" +#include "backend/mmap.h" + +#include + + +/** + * @file + * Keeps all mmaped areas + * + */ + +struct lumiera_mmapings_struct +{ + /** mmaped ranges are kept in an list sorted by the size of the mmaping, might be improved to a tree someday **/ + llist mmaps; + + /** sum of all mmaped areas, areas might be overlapping **/ + // size_t vsz; + + /** + * chunkssize is the smallest granularity which is used for mmapping files, it + * should reflect the intended file usage, that is 'pagesize' for small or non growing + * files and some MB for media files. Must be a 2's exponent of pagesize. + **/ + size_t chunksize; + + LumieraFiledescriptor descriptor; + lumiera_mutex lock; + RESOURCE_HANDLE (rh); +}; + +/** + * initialize mmapings container + * + */ +LumieraMMapings +lumiera_mmapings_init (LumieraMMapings self, LumieraFile file, size_t chunksize); + +/** + * destroy mmapings container and free all resources. + * + */ +LumieraMMapings +lumiera_mmapings_destroy (LumieraMMapings self); + +/** + * allocate and initialize new mmapings container + * + */ +LumieraMMapings +lumiera_mmapings_new (LumieraFile file, size_t chunksize); + +/** + * destroy and free mmapings container and all its resources + * + */ +void +lumiera_mmapings_delete (LumieraMMapings self); + +/** + * acquire a mmap which covers the given range + * @param self mmapings where to search + * @param acquirer list node of the new owner which will registered in the mmap + * @param start begin of the required range + * @param size requested size + * @return MMap object covering the requested range or NULL on error + */ +LumieraMMap +lumiera_mmapings_mmap_acquire (LumieraMMapings self, LumieraFile file, LList acquirer, off_t start, size_t size); + +/** + * release a previously acquired MMap object + * @param self mmapings to which the map belongs + * @param acquirer holding node + * @param map object to be released + */ +void +lumiera_mmapings_release_mmap (LumieraMMapings self, LList acquirer, LumieraMMap map); + +#endif +/* +// Local Variables: +// mode: C +// c-file-style: "gnu" +// indent-tabs-mode: nil +// End: +*/ From f58be32a0563d6a5849fcd528b6aaec416a74b21 Mon Sep 17 00:00:00 2001 From: Christian Thaeter Date: Mon, 19 May 2008 16:37:50 +0200 Subject: [PATCH 39/39] Fixes after rebase * remove _GNU_SOURCE from tests * re-enable C source building in backend --- src/backend/Makefile.am | 15 +++++++-------- tests/backend/test-filedescriptors.c | 2 -- tests/backend/test-filehandles.c | 2 -- 3 files changed, 7 insertions(+), 12 deletions(-) diff --git a/src/backend/Makefile.am b/src/backend/Makefile.am index 45ce555a8..8b33df06a 100644 --- a/src/backend/Makefile.am +++ b/src/backend/Makefile.am @@ -21,14 +21,13 @@ noinst_LIBRARIES += liblumibackend.a liblumibackend_a_CFLAGS = $(CFLAGS) -std=gnu99 -Wall -Werror liblumibackend_a_CPPFLAGS = -I$(top_srcdir)/src/ -liblumibackend_a_SOURCES = \ - $(liblumibackend_a_srcdir)/mediaaccessfacade.cpp - -# $(liblumibackend_a_srcdir)/backend.c \ -# $(liblumibackend_a_srcdir)/file.c \ -# $(liblumibackend_a_srcdir)/filehandle.c \ -# $(liblumibackend_a_srcdir)/filedescriptor.c \ -# $(liblumibackend_a_srcdir)/filehandlecache.c +liblumibackend_a_SOURCES = \ + $(liblumibackend_a_srcdir)/mediaaccessfacade.cpp \ + $(liblumibackend_a_srcdir)/backend.c \ + $(liblumibackend_a_srcdir)/file.c \ + $(liblumibackend_a_srcdir)/filehandle.c \ + $(liblumibackend_a_srcdir)/filedescriptor.c \ + $(liblumibackend_a_srcdir)/filehandlecache.c noinst_HEADERS += \ diff --git a/tests/backend/test-filedescriptors.c b/tests/backend/test-filedescriptors.c index 40f516048..3e14ead1d 100644 --- a/tests/backend/test-filedescriptors.c +++ b/tests/backend/test-filedescriptors.c @@ -18,8 +18,6 @@ along with this program; if not, write to the Free Software Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA. */ -#define _GNU_SOURCE - #include "lib/safeclib.h" #include "backend/backend.h" diff --git a/tests/backend/test-filehandles.c b/tests/backend/test-filehandles.c index 71056144e..17c237d4f 100644 --- a/tests/backend/test-filehandles.c +++ b/tests/backend/test-filehandles.c @@ -18,8 +18,6 @@ along with this program; if not, write to the Free Software Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA. */ -#define _GNU_SOURCE - #include "lib/llist.h" #include "lib/safeclib.h"