[elephant-devel] Re: Berkeley DB error: Cannot allocate memory.

Fri Jan 4 07:14:03 UTC 2008

This is the program:

;;;;
;;;; Playing with Netflix dataset using Elephant to store data
;;;;

(mapc (lambda (x) (asdf:oos 'asdf:load-op x))
	      '(:elephant :iterate :split-sequence))

(defpackage :nelephant
  (:use :cl
	:elephant
	:iterate
	:split-sequence
	))

(in-package :nelephant)

(deftype int32 () '(integer 32))
(defconstant +movies-num+ 17770 "Total number of movies")

(open-store '(:BDB "/home/victor/netflix/db/"))
;(open-store '(:CLSQL (:SQLITE3 "/home/victor/netflix/sqldb.db")))

(defpclass ratings-collection ()
  ((id :type int32 :initarg :id :reader get-id :index t
       :documentation "Object id")
   (ratings :initform (make-btree) :reader ratings
	    :documentation "A hash mapping ids to ratings"))
  (:documentation "An abstract ratings collection"))

(defpclass user (ratings-collection)
  ()
  (:documentation "Movies rated by user with this ID"))

(defpclass movie (ratings-collection)
  ()
  (:documentation "Users rated movie with this ID"))

(defun get-or-create-by-id (id object)
  "Return existing object with given ID or create a new one if it
doesn't exists"
  (or (first (get-instances-by-value object 'id id))
      (make-instance object :id id)))

(declaim (inline get-user))
(defun get-user (id)
  "Return user object with given ID or create a new one"
  (get-or-create-by-id id 'user))

(declaim (inline get-movie))
(defun get-movie (id)
  "Return movie object with given ID or create a new one"
  (get-or-create-by-id id 'movie))

(defun movie-filename (n)
  "Data file with ratings on film N"
  (format nil "download/training_set/mv_~7,'0D.txt" n))

(defun collect-rating-info (movie line)
  "Parse rating data for MOVIE from LINE"
  (destructuring-bind (user rating date)
      (split-sequence #\, line)
    (declare (ignore date))
    (setf (get-value movie (ratings (get-user user)))
	  rating
	  (get-value user (ratings (get-movie movie)))
	  rating)))

(defun import-movie (n)
  "Import ratings information for movie number N."
  (with-transaction ()
      (iter (for line in-file (movie-filename n) using 'read-line)
	    (unless (first-time-p) ; skip first line
	      (collect-rating-info n line)))))

(defun fprinc (obj)
  (princ obj) (princ #\Space)
  (sb-int:flush-standard-output-streams))

(defun import-all-movies ()
  "Import all netflix movies ratings from text files."
  (iter (for i from 1 to +movies-num+)
	(import-movie i)
	(fprinc i)))

On Jan 4, 2008 1:12 AM, Victor Kryukov <victor.kryukov at gmail.com> wrote:
> Hello list,
>
> I've decided to put elephant 0.9.1 under some heavy load test, and
> play with Netflix data set a little bit. The attached program that
> tries to import everything in BerkeleyDB fails when trying to import
> movie file number 8 with the following traceback. Do you have any idea
> what could be the problem?
>
> Thanks,
> Victor.
>
> Berkeley DB error: Cannot allocate memory
>    [Condition of type BDB-DB-ERROR]
>
> Restarts:
>   0: [ABORT-REQUEST] Abort handling SLIME request.
>   1: [ABORT] Exit debugger, returning to top level.
>
> Backtrace:
>   0: ((SB-PCL::FAST-METHOD (SETF GET-VALUE) (T T DB-BDB::BDB-BTREE))
> #<unavailable lambda list>)
>   1: ((LAMBDA NIL))
>   2: ((SB-PCL::FAST-METHOD ELEPHANT::EXECUTE-TRANSACTION
> (DB-BDB::BDB-STORE-CONTROLLER #1="#<...>" . #1#)) #<unavailable
> argument> #<unavailable argument> #<BDB-STORE-CONTROLLER
> /home/victor/netflix/db/> #<CLOSURE (LAMBDA NIL) {1003DC2359}>)
>   3: (IMPORT-ALL-MOVIES)
>   4: (SB-INT:SIMPLE-EVAL-IN-LEXENV (IMPORT-ALL-MOVIES) #<NULL-LEXENV>)
>   5: (SWANK::EVAL-REGION "(import-all-movies)
> " T)
>   6: ((LAMBDA NIL))
>   7: ((SB-PCL::FAST-METHOD SWANK-BACKEND:CALL-WITH-SYNTAX-HOOKS (T))
> #<unused argument> #<unused argument> #<CLOSURE (LAMBDA NIL)
> {100403F959}>)
>   8: (SWANK::CALL-WITH-BUFFER-SYNTAX #<CLOSURE (LAMBDA NIL) {100403F959}>)
>   9: (SB-INT:SIMPLE-EVAL-IN-LEXENV (SWANK:LISTENER-EVAL "(import-all-movies)
> ") #<NULL-LEXENV>)
>  10: ((LAMBDA NIL))
>  11: ((SB-PCL::FAST-METHOD SWANK-BACKEND:CALL-WITH-DEBUGGER-HOOK (T
> T)) #<unused argument> #<unused argument> #<FUNCTION
> SWANK:SWANK-DEBUGGER-HOOK> #<CLOSURE (LAMBDA NIL) {100403F049}>)
>  12: ((LAMBDA NIL))
>  13: ((SB-PCL::FAST-METHOD SWANK-BACKEND:CALL-WITH-DEBUGGER-HOOK (T
> T)) #<unused argument> #<unused argument> #<FUNCTION
> SWANK:SWANK-DEBUGGER-HOOK> #<FUNCTION (LAMBDA NIL) {10028DAF69}>)
>  14: (SWANK::CALL-WITH-REDIRECTED-IO #<SWANK::CONNECTION {1002EFB1E1}>
> #<CLOSURE (LAMBDA NIL) {100403C2F9}>)
>  15: (SWANK::CALL-WITH-CONNECTION #<SWANK::CONNECTION {1002EFB1E1}>
> #<FUNCTION (LAMBDA NIL) {10028DAF69}>)
>  16: (SWANK::HANDLE-REQUEST #<SWANK::CONNECTION {1002EFB1E1}>)
>  17: (SWANK::PROCESS-AVAILABLE-INPUT #<SB-SYS:FD-STREAM for "a socket"
> {1002EE6A01}> #<CLOSURE (LAMBDA NIL) {100403C279}>)
>  18: ((FLET SWANK::HANDLER))
>  19: ((LAMBDA (SWANK-BACKEND::_)) #<unused argument>)
>  20: (SB-IMPL::SUB-SERVE-EVENT NIL NIL NIL)
>  21: (SB-SYS:WAIT-UNTIL-FD-USABLE 0 :INPUT NIL)
>  22: (SB-IMPL::REFILL-INPUT-BUFFER #<SB-SYS:FD-STREAM for "standard
> input" {1002393801}>)
>  23: (SB-IMPL::INPUT-CHAR/UTF-8 #<SB-SYS:FD-STREAM for "standard
> input" {1002393801}> NIL #:EOF-OBJECT)
>  24: (READ-CHAR #<SB-SYS:FD-STREAM for "standard input" {1002393801}>
> NIL #:EOF-OBJECT #<unused argument>)
>  25: (READ-CHAR #<SYNONYM-STREAM :SYMBOL SB-SYS:*STDIN* {100014AEB1}>
> NIL #:EOF-OBJECT #<unused argument>)
>  26: (READ-CHAR #<SYNONYM-STREAM :SYMBOL
> SWANK::*CURRENT-STANDARD-INPUT* {1002384001}> NIL #:EOF-OBJECT
> #<unused argument>)
>  27: (READ-PRESERVING-WHITESPACE #<SYNONYM-STREAM :SYMBOL
> SWANK::*CURRENT-STANDARD-INPUT* {1002384001}> NIL (NIL) T)
>  28: (READ-PRESERVING-WHITESPACE #<SYNONYM-STREAM :SYMBOL
> SWANK::*CURRENT-STANDARD-INPUT* {1002384001}> NIL (NIL) NIL)
>  29: (READ #<SYNONYM-STREAM :SYMBOL SWANK::*CURRENT-STANDARD-INPUT*
> {1002384001}> NIL (NIL) NIL)
>  30: (SB-IMPL::REPL-READ-FORM-FUN #<SYNONYM-STREAM :SYMBOL
> SWANK::*CURRENT-STANDARD-INPUT* {1002384001}> #<unavailable argument>)
>  31: (SB-IMPL::REPL-FUN NIL)
>  32: (SB-IMPL::REPL-FUN NIL)
>  33: ((LAMBDA NIL))
>  34: ((LAMBDA NIL))
>  35: (SB-IMPL::%WITH-REBOUND-IO-SYNTAX #<CLOSURE (LAMBDA NIL) {100386A4D9}>)
>  36: (SB-IMPL::TOPLEVEL-REPL NIL)
>  37: (SB-IMPL::TOPLEVEL-INIT)
>  38: ((LABELS SB-IMPL::RESTART-LISP))
>
>
> --
> Yours Sincerely,
> Victor Kryukov
> Chicago GSB class of 2008
> 773-618-9501
>

-- 
Yours Sincerely,
Victor Kryukov
Chicago GSB class of 2008
773-618-9501