[pve-devel] r4956 - pve-common/trunk

svn-commits at proxmox.com svn-commits at proxmox.com
Tue Aug 10 09:53:42 CEST 2010


Author: dietmar
Date: 2010-08-10 07:53:42 +0000 (Tue, 10 Aug 2010)
New Revision: 4956

Added:
   pve-common/trunk/AtomicFile.pm
   pve-common/trunk/INotify.pm
Modified:
   pve-common/trunk/Makefile
Log:


Added: pve-common/trunk/AtomicFile.pm
===================================================================
--- pve-common/trunk/AtomicFile.pm	                        (rev 0)
+++ pve-common/trunk/AtomicFile.pm	2010-08-10 07:53:42 UTC (rev 4956)
@@ -0,0 +1,18 @@
+package PVE::AtomicFile;
+
+use strict;
+use IO::AtomicFile;
+use vars qw(@ISA);
+
+ at ISA = qw(IO::AtomicFile);
+
+sub new {
+    my $class = shift;
+    my $self = $class->SUPER::new(@_);
+    $self;
+}
+
+
+sub DESTROY {
+    # dont close atomatically (explicit close required to commit changes)
+}

Added: pve-common/trunk/INotify.pm
===================================================================
--- pve-common/trunk/INotify.pm	                        (rev 0)
+++ pve-common/trunk/INotify.pm	2010-08-10 07:53:42 UTC (rev 4956)
@@ -0,0 +1,418 @@
+package PVE::INotify;
+
+use strict;
+use IO::File;
+use IO::Dir;
+use PVE::AtomicFile;
+use File::stat;
+use File::Basename;
+use Fcntl ':flock';
+use PVE::SafeSyslog;
+use Storable qw(dclone);            
+use Linux::Inotify2;
+
+my $ccache;
+my $ccachemap;
+my $inotify;
+my $inotify_pid = 0;
+my $versions;
+
+# to enable cached operation, you need to call 'inotify_init'
+# inotify handles are a limited resource, so use with care (only
+# enable the cache if you really need it)
+
+# Note: please close the inotify handle after you fork
+
+my $shadowfiles = {
+    '/etc/network/interfaces' => '/etc/network/interfaces.new',
+};
+
+sub ccache_default_writer {
+    my ($filename, $data) = @_;
+
+    die "undefined config writer for '$filename' :ERROR";
+}
+
+sub ccache_default_parser {
+    my ($filename, $srcfd) = @_;
+
+    die "undefined config reader for '$filename' :ERROR";
+}
+
+sub ccache_compute_diff {
+    my ($filename, $shadow) = @_;
+
+    my $diff = '';
+
+    open (TMP, "diff -b -N -u '$filename' '$shadow'|");
+	
+    while (my $line = <TMP>) {
+	$diff .= $line;
+    }
+
+    close (TMP);
+
+    $diff = undef if !$diff;
+
+    return $diff;
+}
+
+sub write_file {
+    my ($filename, $data, $full) = @_;
+
+    $filename = $ccachemap->{$filename} if defined ($ccachemap->{$filename});
+
+    die "file '$filename' not added :ERROR" if !defined ($ccache->{$filename});
+
+    my $writer = $ccache->{$filename}->{writer};
+
+    my $realname = $filename;
+
+    my $shadow;
+    if ($shadow = $shadowfiles->{$filename}) {
+	$realname = $shadow;
+    }
+
+    my $fh = PVE::AtomicFile->open($realname, "w") ||
+	die "unable to open file '$realname' for writing - $! :ERROR";
+
+    my $res;
+
+    eval {
+	$res = &$writer ($filename, $fh, $data);
+    };
+
+    $ccache->{$filename}->{version} = undef;
+
+    my $err = $@;
+    $fh->detach() if $err;
+    $fh->close(1);
+
+    die $err if $err;
+
+    my $diff;
+    if ($shadow && $full) {
+	$diff = ccache_compute_diff ($filename, $shadow);
+    }
+
+    if ($full) {
+	return { data => $res, changes => $diff };
+    }
+
+    return $res;
+}
+
+sub update_file {
+    my ($filename, $data, @args) = @_;
+
+    $filename = $ccachemap->{$filename} if defined ($ccachemap->{$filename});
+
+    my $update = $ccache->{$filename}->{update};
+
+    die "unable to update/merge data" if !$update;
+
+    my $lkfn = "$filename.lock";
+
+    if (!open (FLCK, ">>$lkfn")) {
+	die "unable to open lock file '$lkfn' - $?";
+    }
+
+    if (!flock (FLCK, LOCK_EX)) {
+	close (FLCK);
+	die "unable to aquire lock for file '$lkfn' - $?";
+    }
+
+    my $newdata;
+
+    eval {
+
+	my $olddata = read_file ($filename);
+
+	if ($data) {
+	    my $res = &$update ($filename, $olddata, $data, @args);
+	    if (defined ($res)) {
+		$newdata = write_file ($filename, $res);
+	    } else {
+		$newdata = $olddata;
+	    }
+	} else {
+	    $newdata = $olddata;
+	}
+    };
+
+    my $err = $@;
+
+    close (FLCK);
+
+    die $err if $err;
+
+    return $newdata;
+}
+
+sub discard_changes {
+    my ($filename, $full) = @_;
+
+    $filename = $ccachemap->{$filename} if defined ($ccachemap->{$filename});
+
+    die "file '$filename' not added :ERROR" if !defined ($ccache->{$filename});
+
+    if (my $copy = $shadowfiles->{$filename}) {
+	unlink $copy;
+    }
+
+    return read_file ($filename, $full);
+}
+
+sub read_file {
+    my ($filename, $full) = @_;
+
+    my $parser;
+
+# fixme: allow regex to register parsers
+#    if ($filename =~ m|^/etc/qemu-server/\d+\.conf$|) {
+#	$parser = \&read_qmconfig;
+#    } elsif ($filename =~ m|^/etc/vz/conf/\d+\.conf$|) {
+#	$parser = \&read_vzconfig;
+#    } else {
+
+    $filename = $ccachemap->{$filename} if defined ($ccachemap->{$filename});
+
+    die "file '$filename' not added :ERROR" if !defined ($ccache->{$filename});
+
+    $parser = $ccache->{$filename}->{parser};
+ 
+    my $fd;
+    my $shadow;
+
+    poll() if $inotify; # read new inotify events
+
+    $versions->{$filename} = 0 if !defined ($versions->{$filename});
+
+    my $cver = $versions->{$filename};
+
+    if (my $copy = $shadowfiles->{$filename}) {
+	if ($fd = IO::File->new ($copy, "r")) {
+	    $shadow = $copy;
+	} else {
+	    $fd = IO::File->new ($filename, "r");
+	}
+    } else {
+	$fd = IO::File->new ($filename, "r");
+    }
+
+    my $acp = $ccache->{$filename}->{always_call_parser};
+
+    if (!$fd) {
+	$ccache->{$filename}->{version} = undef;
+	$ccache->{$filename}->{data} = undef; 
+	$ccache->{$filename}->{diff} = undef;
+	return undef if !$acp;
+    }
+
+    my $noclone = $ccache->{$filename}->{noclone};
+
+    # file unchanged?
+    if (!$ccache->{$filename}->{nocache} &&
+	$inotify && $versions->{$filename} &&
+	defined ($ccache->{$filename}->{data}) &&
+	defined ($ccache->{$filename}->{version}) &&
+	($ccache->{$filename}->{readonce} ||
+	 ($ccache->{$filename}->{version} == $versions->{$filename}))) {
+
+	my $ret;
+	if (!$noclone && ref ($ccache->{$filename}->{data})) {
+	    $ret->{data} = dclone ($ccache->{$filename}->{data});
+	} else {
+	    $ret->{data} = $ccache->{$filename}->{data};
+	}
+	$ret->{changes} = $ccache->{$filename}->{diff};
+	
+	return $full ? $ret : $ret->{data};
+    }
+
+    my $diff;
+
+    if ($shadow) {
+	$diff = ccache_compute_diff ($filename, $shadow);
+    }
+
+    my $res = &$parser ($filename, $fd);
+
+    if (!$ccache->{$filename}->{nocache}) {
+	$ccache->{$filename}->{version} = $cver;
+    }
+
+    # we cache data with references, so we always need to
+    # dclone this data. Else the original data may get
+    # modified.
+    $ccache->{$filename}->{data} = $res;
+
+    # also store diff
+    $ccache->{$filename}->{diff} = $diff;
+
+    my $ret;
+    if (!$noclone && ref ($ccache->{$filename}->{data})) {
+	$ret->{data} = dclone ($ccache->{$filename}->{data});
+    } else {
+	$ret->{data} = $ccache->{$filename}->{data};
+    }
+    $ret->{changes} = $ccache->{$filename}->{diff};
+
+    return $full ? $ret : $ret->{data};
+}    
+
+sub add_file {
+    my ($id, $filename, $parser, $writer, $update, %options) = @_;
+
+    die "file '$filename' already added :ERROR" if defined ($ccache->{$filename});
+    die "ID '$id' already used :ERROR" if defined ($ccachemap->{$id});
+
+    $ccachemap->{$id} = $filename;
+    $ccache->{$filename}->{id} = $id;
+
+    $ccache->{$filename}->{parser} = $parser || \&ccache_default_parser;
+    $ccache->{$filename}->{writer} = $writer || \&ccache_default_writer;
+    $ccache->{$filename}->{update} = $update;
+
+    foreach my $opt (keys %options) {
+	my $v = $options{$opt};
+	if ($opt eq 'readonce') {
+	    $ccache->{$filename}->{$opt} = $v;
+	} elsif ($opt eq 'nocache') {
+	    $ccache->{$filename}->{$opt} = $v;
+	} elsif ($opt eq 'noclone') {
+	    # noclone flag for large read-only data chunks like aplinfo
+	    $ccache->{$filename}->{$opt} = $v;
+	} elsif ($opt eq 'always_call_parser') {
+	    # when set, we call parser even when the file does not exists.
+	    # this allows the parser to return some default
+	    $ccache->{$filename}->{$opt} = $v;
+	} else {
+	    die "internal error - unsupported option '$opt'";
+	}
+    }
+}
+
+sub poll {
+    return if !$inotify;
+
+    if ($inotify_pid != $$) {
+	syslog ('err', "got inotify poll request in wrong process - disabling inotify");
+	$inotify = undef;
+    } else {
+	1 while $inotify && $inotify->poll;
+    }
+}
+
+sub flushcache {
+    foreach my $filename (keys %$ccache) {
+	$ccache->{$filename}->{version} = undef;
+	$ccache->{$filename}->{data} = undef;
+	$ccache->{$filename}->{diff} = undef;
+    }
+}
+
+sub inotify_close {
+    $inotify = undef;
+}
+
+sub inotify_init {
+
+    die "only one inotify instance allowed" if $inotify;
+
+    $inotify =  Linux::Inotify2->new()
+	|| die "Unable to create new inotify object: $!";
+
+    $inotify->blocking (0);
+
+    $versions = {};
+
+    my $dirhash = {};
+    foreach my $fn (keys %$ccache) {
+	my $dir = dirname ($fn);
+	my $base = basename ($fn);
+
+	$dirhash->{$dir}->{$base} = $fn;
+
+	if (my $sf = $shadowfiles->{$fn}) {
+	    $base = basename ($sf);
+	    $dir = dirname ($sf);
+	    $dirhash->{$dir}->{$base} = $fn; # change version of original file!
+	}
+    }
+
+    # also get versions of qemu and openvz config files
+    $dirhash->{"/etc/qemu-server"}->{_regex} = '\d+\.conf';
+    $dirhash->{"/etc/vz/conf"}->{_regex} = '\d+\.conf';
+
+    $inotify_pid = $$;
+
+    foreach my $dir (keys %$dirhash) {
+
+	my $evlist = IN_MODIFY|IN_ATTRIB|IN_MOVED_FROM|IN_MOVED_TO|IN_DELETE|IN_CREATE;
+	$inotify->watch ($dir, $evlist, sub {
+	    my $e = shift;
+	    my $name = $e->name;
+
+	    if ($inotify_pid != $$) {
+		syslog ('err', "got inotify event in wrong process");
+	    }
+
+	    if ($e->IN_ISDIR || !$name) {
+		return;
+	    }
+
+	    if ($e->IN_Q_OVERFLOW) {
+		syslog ('info', "got inotify overflow - flushing cache");
+		flushcache();
+		return;
+	    }
+
+	    if ($e->IN_UNMOUNT) {
+		syslog ('err', "got 'unmount' event on '$name' - disabling inotify");
+		$inotify = undef;
+	    }
+	    if ($e->IN_IGNORED) { 
+		syslog ('err', "got 'ignored' event on '$name' - disabling inotify");
+		$inotify = undef;
+	    }
+
+	    my $re = $dirhash->{$dir}->{_regex};
+	    if ($re && ($name =~ m|^$re$|)) {
+
+		my $fn = "$dir/$name";
+		$versions->{$fn}++;
+		#print "VERSION:$fn:$versions->{$fn}\n";
+
+	    } elsif (my $fn = $dirhash->{$dir}->{$name}) {
+
+		$versions->{$fn}++;
+		#print "VERSION:$fn:$versions->{$fn}\n";
+	    }
+	});
+    }
+
+    foreach my $dir (keys %$dirhash) {
+	foreach my $name (keys %{$dirhash->{$dir}}) {
+	    if ($name eq '_regex') {
+		my $re = $dirhash->{$dir}->{_regex};
+		if (my $fd = IO::Dir->new ($dir)) {
+		    while (defined(my $de = $fd->read)) { 
+			if ($de =~ m/^$re$/) {
+			    my $fn = "$dir/$de";
+			    $versions->{$fn}++; # init with version
+			    #print "init:$fn:$versions->{$fn}\n";
+			}
+		    }
+		}
+	    } else {
+		my $fn = $dirhash->{$dir}->{$name};
+		$versions->{$fn}++; # init with version
+		#print "init:$fn:$versions->{$fn}\n";
+	    }
+	}
+    }
+
+}
+
+1;

Modified: pve-common/trunk/Makefile
===================================================================
--- pve-common/trunk/Makefile	2010-08-10 07:23:54 UTC (rev 4955)
+++ pve-common/trunk/Makefile	2010-08-10 07:53:42 UTC (rev 4956)
@@ -20,6 +20,8 @@
 LIB_SOURCES=			\
 	JSONSchema.pm		\
 	SafeSyslog.pm		\
+	AtomicFile.pm		\
+	INotify.pm		\
 	Exception.pm
 
 all: ${DEB}



More information about the pve-devel mailing list