From me at swizard.info Tue Sep 8 18:47:59 2009 From: me at swizard.info (Alexey Voznyuk) Date: Tue, 08 Sep 2009 22:47:59 +0400 Subject: [cl-prevalence-devel] [feature proposal][patch] :external-format for file operations Message-ID: <4AA6A6DF.2030300@swizard.info> Hello! Current cl-prevalence version does not allow to set specific :external-format when reading or writing files. This is very uncomfortable when current working encoding in application differs from the system locale's one: cl recodes strings on serializing. Please check this patchset implementing the feature (cumulative with my previous bugfixes for "serialize-sexp-external" and "backup"). Maybe there are some crossplatform-related changes required. % hg diff diff -r 1edf5cd93bb9 src/prevalence.lisp --- a/src/prevalence.lisp Sun Jul 26 11:10:13 2009 +0200 +++ b/src/prevalence.lisp Tue Sep 08 22:44:04 2009 +0400 @@ -103,7 +103,11 @@ (transaction-hook ;; type function :accessor get-transaction-hook :initarg :transaction-hook - :initform #'identity)) + :initform #'identity) + (external-format ;; external encoding for file operations + :reader get-external-format + :initarg :external-format + :initform :utf-8)) (:documentation "Base Prevalence system implementation object")) (defclass guarded-prevalence-system (prevalence-system) @@ -156,7 +160,8 @@ (setf transaction-log-stream (open (get-transaction-log system) :direction :output :if-does-not-exist :create - :if-exists :append))))) + :if-exists :append + :external-format (get-external-format system)))))) (defmethod close-open-streams ((system prevalence-system) &key abort) "Close all open stream associated with system (optionally aborting operations in progress)" @@ -242,7 +247,7 @@ :type (get-file-extension system)) snapshot))) (with-open-file (out snapshot - :direction :output :if-does-not-exist :create :if-exists :supersede) + :direction :output :if-does-not-exist :create :if-exists :supersede :external-format (get-external-format system)) (funcall (get-serializer system) (get-root-objects system) out (get-serialization-state system))) (when (probe-file transaction-log) (copy-file transaction-log (merge-pathnames (make-pathname :name (get-transaction-log-filename system timetag) @@ -273,7 +278,7 @@ (clrhash (get-root-objects system)) (close-open-streams system) (when (probe-file (get-snapshot system)) - (with-open-file (in (get-snapshot system) :direction :input) + (with-open-file (in (get-snapshot system) :direction :input :external-format (get-external-format system)) (setf (get-root-objects system) (funcall (get-deserializer system) in (get-serialization-state system))))) (when (probe-file (get-transaction-log system)) (let ((position 0)) @@ -284,7 +289,7 @@ condition) (truncate-file (get-transaction-log system) position) (return-from restore)))) - (with-open-file (in (get-transaction-log system) :direction :input) + (with-open-file (in (get-transaction-log system) :direction :input :external-format (get-external-format system)) (loop (let ((transaction (funcall (get-deserializer system) in (get-serialization-state system)))) (setf position (file-position in)) @@ -310,7 +315,7 @@ (defmethod backup ((system guarded-prevalence-system) &key directory) "Do a backup on a system controlled by a guard" (funcall (get-guard system) - #'(lambda () (call-next-method system directory)))) + #'(lambda () (call-next-method system :directory directory)))) (defmethod restore ((system guarded-prevalence-system)) "Restore a system controlled by a guard" @@ -343,8 +348,8 @@ (buffer (make-string 4096)) (index 0) (read-count 0)) - (with-open-file (in file :direction :input) - (with-open-file (out tmp-file :direction :output :if-exists :overwrite :if-does-not-exist :create) + (with-open-file (in file :direction :input :external-format (get-external-format system)) + (with-open-file (out tmp-file :direction :output :if-exists :overwrite :if-does-not-exist :create :external-format (get-external-format system)) (when (> position (file-length in)) (return-from truncate-file)) (loop (when (= index position) (return)) @@ -360,8 +365,8 @@ (defun copy-file (source target) (let ((buffer (make-string 4096)) (read-count 0)) - (with-open-file (in source :direction :input) - (with-open-file (out target :direction :output :if-exists :overwrite :if-does-not-exist :create) + (with-open-file (in source :direction :input :external-format (get-external-format system)) + (with-open-file (out target :direction :output :if-exists :overwrite :if-does-not-exist :create :external-format (get-external-format system)) (loop (setf read-count (read-sequence buffer in)) (write-sequence buffer out :end read-count) diff -r 1edf5cd93bb9 src/serialization/sexp.lisp --- a/src/serialization/sexp.lisp Sun Jul 26 11:10:13 2009 +0200 +++ b/src/serialization/sexp.lisp Tue Sep 08 22:44:04 2009 +0400 @@ -145,8 +145,9 @@ (write-string " . " stream) (serialize-sexp-internal (slot-value object slot) stream serialization-state) (write-string ")" stream)) - serializable-slots)) - (write-string " ) )" stream))))) + serializable-slots) + (write-string " )" stream)) + (write-string " )" stream))))) ;;; objects (defmethod serialize-sexp-internal ((object standard-object) stream serialization-state) From me at swizard.info Tue Sep 8 18:53:07 2009 From: me at swizard.info (Alexey Voznyuk) Date: Tue, 08 Sep 2009 22:53:07 +0400 Subject: [cl-prevalence-devel] [feature proposal][patch] :external-format for file operations In-Reply-To: <4AA6A6DF.2030300@swizard.info> References: <4AA6A6DF.2030300@swizard.info> Message-ID: <4AA6A813.9030004@swizard.info> Alexey Voznyuk wrote: > Hello! > > Current cl-prevalence version does not allow to set specific > :external-format when reading or writing files. This is very > uncomfortable when current working encoding in application differs from > the system locale's one: cl recodes strings on serializing. > > > Please check this patchset implementing the feature (cumulative with > my previous bugfixes for "serialize-sexp-external" and "backup"). > Maybe there are some crossplatform-related changes required. > > [snip] Here is the copy: http://swizard.info/misc/cl-prevalence.patch From sky at viridian-project.de Thu Sep 10 08:40:26 2009 From: sky at viridian-project.de (Leslie P. Polzer) Date: Thu, 10 Sep 2009 10:40:26 +0200 (CEST) Subject: [cl-prevalence-devel] [feature proposal][patch] :external-format for file operations In-Reply-To: <4AA6A6DF.2030300@swizard.info> References: <4AA6A6DF.2030300@swizard.info> Message-ID: <2dbc8755dc134539fffed036ac0dec74.squirrel@mail.stardawn.org> Hi Alexey, thanks for your proposal. However to integrate this it would be nice to have at least basic test cases. Can you provide some as part of your patch? Leslie -- http://www.linkedin.com/in/polzer From me at swizard.info Thu Sep 24 23:55:07 2009 From: me at swizard.info (Alexey Voznyuk) Date: Fri, 25 Sep 2009 03:55:07 +0400 Subject: [cl-prevalence-devel] [feature proposal][patch] :external-format for file operations In-Reply-To: <2dbc8755dc134539fffed036ac0dec74.squirrel@mail.stardawn.org> References: <4AA6A6DF.2030300@swizard.info> <2dbc8755dc134539fffed036ac0dec74.squirrel@mail.stardawn.org> Message-ID: <4ABC06DB.1050707@swizard.info> Hi Leslie! Leslie P. Polzer wrote: > Hi Alexey, > > thanks for your proposal. However to integrate this it would > be nice to have at least basic test cases. > > Can you provide some as part of your patch? > > Leslie > I'm really sorry about such reply delay. I hope I didn't cause any troubles for you with it. However you are right about tests, I have a bug in my previous patch :) Here is the correct version with some tests included: http://swizard.info/misc/cl-prevalence-2.patch From me at swizard.info Fri Sep 25 00:28:50 2009 From: me at swizard.info (Alexey Voznyuk) Date: Fri, 25 Sep 2009 04:28:50 +0400 Subject: [cl-prevalence-devel] Keeping state during transactions Message-ID: <4ABC0EC2.80804@swizard.info> Hello! I've faced annoying cl-prevalence behavior which I don't know how to deal with. For example, consider this code that I want to run within a transaction: (defun tx-add-item (store index item) (declare (ignore store)) (setf (gethash item index) t)) Current cl-prevalence implementation invokes (reset serialization-state) before each (serialize-*) call. So as long as I insert new items into the index, each log entry becomes bigger and bigger because it needs to deeply serialize "index" parameter with all its elements, including the new ones. The problem that the index can contains thousands elements, and each of them can be a complex object itself with maybe more index tables. In such case a single transaction can consume a lot of resource. My proposal is to not reset serialization-state before each transaction serialize step but to keep it until full snapshot is occurred. For me it seems to work correctly, but maybe I miss something? From sky at viridian-project.de Fri Sep 25 09:24:16 2009 From: sky at viridian-project.de (Leslie P. Polzer) Date: Fri, 25 Sep 2009 11:24:16 +0200 (CEST) Subject: [cl-prevalence-devel] Keeping state during transactions In-Reply-To: <4ABC0EC2.80804@swizard.info> References: <4ABC0EC2.80804@swizard.info> Message-ID: <7a4738fad2d52081e8e20fd70084ea45.squirrel@mail.stardawn.org> Alexey Voznyuk wrote: > My proposal is to not reset serialization-state before each > transaction serialize step but to keep it until full snapshot is > occurred. For me it seems to work correctly, but maybe I miss something? Here's a patch I wrote and use since; it wasn't made for performance but for correctness in maintaining object references. I intended to merge it in but haven't got around to write tests for it yet. Perhaps you'd like to do that? Leslie -- http://www.linkedin.com/in/polzer -------------- next part -------------- A non-text attachment was scrubbed... Name: prevalence-object-xrefs.diff Type: text/x-patch Size: 9427 bytes Desc: not available URL: From sky at viridian-project.de Fri Sep 25 09:27:58 2009 From: sky at viridian-project.de (Leslie P. Polzer) Date: Fri, 25 Sep 2009 11:27:58 +0200 (CEST) Subject: [cl-prevalence-devel] [feature proposal][patch] :external-format for file operations In-Reply-To: <4ABC06DB.1050707@swizard.info> References: <4AA6A6DF.2030300@swizard.info> <2dbc8755dc134539fffed036ac0dec74.squirrel@mail.stardawn.org> <4ABC06DB.1050707@swizard.info> Message-ID: <8cc46061c322b81f3a0f5f98a22bb1ae.squirrel@mail.stardawn.org> Hi Alexey > I'm really sorry about such reply delay. I hope I didn't cause any > troubles for you with it. Not at all. I haven't merged your patches yet either. Hope I'll get around to that soon. Thanks for the update with tests! Leslie -- http://www.linkedin.com/in/polzer From me at swizard.info Fri Sep 25 11:26:45 2009 From: me at swizard.info (Alexey Voznyuk) Date: Fri, 25 Sep 2009 15:26:45 +0400 Subject: [cl-prevalence-devel] Keeping state during transactions In-Reply-To: <7a4738fad2d52081e8e20fd70084ea45.squirrel@mail.stardawn.org> References: <4ABC0EC2.80804@swizard.info> <7a4738fad2d52081e8e20fd70084ea45.squirrel@mail.stardawn.org> Message-ID: <4ABCA8F5.20805@swizard.info> Leslie P. Polzer wrote: > Alexey Voznyuk wrote: >> My proposal is to not reset serialization-state before each >> transaction serialize step but to keep it until full snapshot is >> occurred. For me it seems to work correctly, but maybe I miss something? >> > > Here's a patch I wrote and use since; it wasn't made for performance > but for correctness in maintaining object references. > > I intended to merge it in but haven't got around to write tests for it > yet. Perhaps you'd like to do that? > > Leslie > Sure, no problems. From me at swizard.info Sat Sep 26 01:56:59 2009 From: me at swizard.info (Alexey Voznyuk) Date: Sat, 26 Sep 2009 05:56:59 +0400 Subject: [cl-prevalence-devel] 20090926 cummulative patchset Message-ID: <4ABD74EB.6000201@swizard.info> Hello! Here is what I have done since 20090828 in 'hg diff' format: http://swizard.info/misc/cl-prevalence-3.patch It is going slightly uncomfortable for me to maintain patchfiles, maybe I should make a mercurial branch or something? Detailed changelog: 1. SEXP standard-object serializing: fixed a bug with no-slots objects. 2. Backup method for guarded-prevalence-system: key parameter bugfix. 3. Files external format support for prevalence-system with tests available (test/test-external-format.lisp). 4. Moved from 'copy-file'+'delete-file' to 'rename-file' in snapshot method. Have no idea why it was like that :) 5. Removed reset serialization context invocations in serialize-sexp, deserialize-sexp, serialize-xml and deserialize-xml. Context reset moved to 'snapshot' method. 6. test/test-complex-serialize.lisp: emulation of complex datasets processing under heavy load. From me at swizard.info Sat Sep 26 02:15:33 2009 From: me at swizard.info (Alexey Voznyuk) Date: Sat, 26 Sep 2009 06:15:33 +0400 Subject: [cl-prevalence-devel] Keeping state during transactions In-Reply-To: <7a4738fad2d52081e8e20fd70084ea45.squirrel@mail.stardawn.org> References: <4ABC0EC2.80804@swizard.info> <7a4738fad2d52081e8e20fd70084ea45.squirrel@mail.stardawn.org> Message-ID: <4ABD7945.1080702@swizard.info> Leslie P. Polzer wrote: > Alexey Voznyuk wrote: >> My proposal is to not reset serialization-state before each >> transaction serialize step but to keep it until full snapshot is >> occurred. For me it seems to work correctly, but maybe I miss something? >> > > Here's a patch I wrote and use since; it wasn't made for performance > but for correctness in maintaining object references. > > I intended to merge it in but haven't got around to write tests for it > yet. Perhaps you'd like to do that? > > Leslie > There is something wrong with this patch. It seems it targets the old cl-prevalence release (cvs one), where serialization.lisp is not splitted yet. But it doesn't matter, I see the following problems there: - the patch seems incomplete (I didn't find where *txn-state* is defined, there are only use cases) - I didn't get what *txn-state* is for. All that it does is doubling 'known-object-id' / 'set-known-object' functions. - There are only xml serialization related fixes, nothing for sexp. Please check my latest 20090926 cumulative patch where I only removed serialization state clearing on each transaction. There are some tests included. From sky at viridian-project.de Sat Sep 26 07:37:23 2009 From: sky at viridian-project.de (Leslie P. Polzer) Date: Sat, 26 Sep 2009 09:37:23 +0200 (CEST) Subject: [cl-prevalence-devel] 20090926 cummulative patchset In-Reply-To: <4ABD74EB.6000201@swizard.info> References: <4ABD74EB.6000201@swizard.info> Message-ID: <279a7b0a4566782455ee48292490c2aa.squirrel@mail.stardawn.org> Alexey Voznyuk wrote: > Here is what I have done since 20090828 in 'hg diff' format: > http://swizard.info/misc/cl-prevalence-3.patch > It is going slightly uncomfortable for me to maintain patchfiles, > maybe I should make a mercurial branch or something? Yes, that would be great! Leslie -- http://www.linkedin.com/in/polzer From sky at viridian-project.de Sat Sep 26 07:40:51 2009 From: sky at viridian-project.de (Leslie P. Polzer) Date: Sat, 26 Sep 2009 09:40:51 +0200 (CEST) Subject: [cl-prevalence-devel] Keeping state during transactions In-Reply-To: <4ABD7945.1080702@swizard.info> References: <4ABC0EC2.80804@swizard.info> <7a4738fad2d52081e8e20fd70084ea45.squirrel@mail.stardawn.org> <4ABD7945.1080702@swizard.info> Message-ID: <05ccdb55b06e272b3ba427dedf9655c6.squirrel@mail.stardawn.org> Alexey Voznyuk wrote: > There is something wrong with this patch. > > It seems it targets the old cl-prevalence release (cvs one), where > serialization.lisp is not splitted yet. But it doesn't matter, I see the > following problems there: > - the patch seems incomplete (I didn't find where *txn-state* is > defined, there are only use cases) The following hunks seem to be missing: diff --git a/lib/src/cl-prevalence/src/package.lisp b/lib/src/cl-prevalence/src/package.lisp index 3b924e9..973e9ca 100644 --- a/lib/src/cl-prevalence/src/package.lisp +++ b/lib/src/cl-prevalence/src/package.lisp @@ -13,6 +13,7 @@ (defpackage :s-serialization (:use :cl) (:export + #:*txn-state* #:serializable-slots #:serialize-xml #:serialize-sexp #:deserialize-xml #:deserialize-sexp diff --git a/lib/src/cl-prevalence/src/prevalence.lisp b/lib/src/cl-prevalence/src/prevalence.lisp index 7d21e2a..acb51d8 100644 --- a/lib/src/cl-prevalence/src/prevalence.lisp +++ b/lib/src/cl-prevalence/src/prevalence.lisp @@ -213,7 +213,9 @@ (defmethod log-transaction ((system prevalence-system) (transaction transaction)) "Log transaction for system" - (let ((out (get-transaction-log-stream system))) + (let ((*txn-state* (make-hash-table)) + (out (get-transaction-log-stream system))) + (declare (special *txn-state*)) (funcall (get-serializer system) transaction out (get-serialization-state system)) (terpri out) (finish-output out))) > - I didn't get what *txn-state* is for. All that it does is > doubling 'known-object-id' / 'set-known-object' functions. *txn-state* holds the transaction-specific state. The old model only has transaction states via a system-global variable that gets reset before every transaction. This new code moves this state to *txn-state* and uses the system-global variable as a true global state that does not get reset. Does that make things clear? > - There are only xml serialization related fixes, nothing for sexp. Sorry about that, it's missing indeed. One of the reasons it's not merged in yet... > Please check my latest 20090926 cumulative patch where I only > removed serialization state clearing on each transaction. There are some > tests included. I don't think this will suffice. Can your code handle cycles in the object graph? Leslie -- http://www.linkedin.com/in/polzer From me at swizard.info Sat Sep 26 11:13:01 2009 From: me at swizard.info (Alexey Voznyuk) Date: Sat, 26 Sep 2009 15:13:01 +0400 Subject: [cl-prevalence-devel] Keeping state during transactions In-Reply-To: <05ccdb55b06e272b3ba427dedf9655c6.squirrel@mail.stardawn.org> References: <4ABC0EC2.80804@swizard.info> <7a4738fad2d52081e8e20fd70084ea45.squirrel@mail.stardawn.org> <4ABD7945.1080702@swizard.info> <05ccdb55b06e272b3ba427dedf9655c6.squirrel@mail.stardawn.org> Message-ID: <4ABDF73D.2060405@swizard.info> Leslie P. Polzer wrote: > [snip] >> - I didn't get what *txn-state* is for. All that it does is >> doubling 'known-object-id' / 'set-known-object' functions. >> > > *txn-state* holds the transaction-specific state. The old model > only has transaction states via a system-global variable that > gets reset before every transaction. This new code moves this > state to *txn-state* and uses the system-global variable as a > true global state that does not get reset. > > Does that make things clear? > No :) The only purpose of 'state' in serialization is to maintain the objects table already serialized, so we can reference them instead of full persistence procedure. The old cl-prevalence implementation keeps that state only during one 'transaction', so the following transactions need to deeply serialize objects, that were already been serialized during previous transactions. My purpose to maintain that objects table until full snapshot is occurred. I don't see what *txn-state* is for. It only copies the old state behavior. > [snip] >> Please check my latest 20090926 cumulative patch where I only >> removed serialization state clearing on each transaction. There are some >> tests included. >> > > I don't think this will suffice. Can your code handle cycles in > the object graph? > Sure, no problems here. The references to the already known objects are not serialized. Check test/test-complex-serialize.lisp in my recent cl-prevalence-3.patch, there are tests for complex objects structure with cycles and several indexes. > Leslie > > From me at swizard.info Sat Sep 26 11:16:23 2009 From: me at swizard.info (Alexey Voznyuk) Date: Sat, 26 Sep 2009 15:16:23 +0400 Subject: [cl-prevalence-devel] 20090926 cummulative patchset In-Reply-To: <279a7b0a4566782455ee48292490c2aa.squirrel@mail.stardawn.org> References: <4ABD74EB.6000201@swizard.info> <279a7b0a4566782455ee48292490c2aa.squirrel@mail.stardawn.org> Message-ID: <4ABDF807.80305@swizard.info> Leslie P. Polzer wrote: > Alexey Voznyuk wrote: > > >> Here is what I have done since 20090828 in 'hg diff' format: >> http://swizard.info/misc/cl-prevalence-3.patch >> It is going slightly uncomfortable for me to maintain patchfiles, >> maybe I should make a mercurial branch or something? >> > > Yes, that would be great! > Huh, what I should do exactly? :) I can create a branch right here: http://bitbucket.org/skypher/cl-prevalence/ or I should fork the project and copy it to my own mercurial installation? > Leslie > > From sky at viridian-project.de Mon Sep 28 08:38:45 2009 From: sky at viridian-project.de (Leslie P. Polzer) Date: Mon, 28 Sep 2009 10:38:45 +0200 (CEST) Subject: [cl-prevalence-devel] 20090926 cummulative patchset In-Reply-To: <4ABDF807.80305@swizard.info> References: <4ABD74EB.6000201@swizard.info> <279a7b0a4566782455ee48292490c2aa.squirrel@mail.stardawn.org> <4ABDF807.80305@swizard.info> Message-ID: Alexey Voznyuk wrote: > Huh, what I should do exactly? :) I can create a branch right here: > http://bitbucket.org/skypher/cl-prevalence/ or I should fork the project > and copy it to my own mercurial installation? Create a fork on Bitbucket and push all your changes to it. I will pull/transplant them as soon as I have reviewed them. Please make at least one commit per feature. Don't forget to post the address here. :) Leslie -- http://www.linkedin.com/in/polzer