From psmith at common-lisp.net Tue Apr 17 04:22:57 2007 From: psmith at common-lisp.net (psmith at common-lisp.net) Date: Tue, 17 Apr 2007 00:22:57 -0400 (EDT) Subject: [nio-cvs] r108 - branches/home/psmith/restructure/src/event Message-ID: <20070417042257.A19B61B000@common-lisp.net> Author: psmith Date: Tue Apr 17 00:22:56 2007 New Revision: 108 Modified: branches/home/psmith/restructure/src/event/epoll.lisp Log: Print out interruptions Modified: branches/home/psmith/restructure/src/event/epoll.lisp ============================================================================== --- branches/home/psmith/restructure/src/event/epoll.lisp (original) +++ branches/home/psmith/restructure/src/event/epoll.lisp Tue Apr 17 00:22:56 2007 @@ -89,7 +89,9 @@ (-1 (let ((errno (get-errno))) (if (eql errno 4) ;EINTR - interrupted by a system call - (return nil) + (progn + (format t "epoll-wait interrupted~%") + (return nil)) (progn (format t "-1 returned from epoll-wait, errno:") (perror) From psmith at common-lisp.net Tue Apr 17 04:26:33 2007 From: psmith at common-lisp.net (psmith at common-lisp.net) Date: Tue, 17 Apr 2007 00:26:33 -0400 (EDT) Subject: [nio-cvs] r109 - in branches/home/psmith/restructure/src: io nio-logger protocol/yarpc Message-ID: <20070417042633.4892821051@common-lisp.net> Author: psmith Date: Tue Apr 17 00:26:29 2007 New Revision: 109 Modified: branches/home/psmith/restructure/src/io/async-fd.lisp branches/home/psmith/restructure/src/io/nio-package.lisp branches/home/psmith/restructure/src/io/nio-server.lisp branches/home/psmith/restructure/src/nio-logger/nio-logger.lisp branches/home/psmith/restructure/src/protocol/yarpc/nio-yarpc-package.lisp branches/home/psmith/restructure/src/protocol/yarpc/yarpc-client-state-machine.lisp branches/home/psmith/restructure/src/protocol/yarpc/yarpc-state-machine.lisp Log: Remove +process-jobs-inline+ as can't work like this. Added timeout mechanism Modified: branches/home/psmith/restructure/src/io/async-fd.lisp ============================================================================== --- branches/home/psmith/restructure/src/io/async-fd.lisp (original) +++ branches/home/psmith/restructure/src/io/async-fd.lisp Tue Apr 17 00:26:29 2007 @@ -62,6 +62,9 @@ ;;Implement this in concrete SM for read (defgeneric process-write (async-fd)) +;;Implement this in concrete SM for timeout processing +(defgeneric process-timeout (async-fd)) + ;;SM factory (defun create-state-machine(sm-type read-fd write-fd socket) (let ((sm (make-instance sm-type :read-fd read-fd :write-fd write-fd :socket socket))) Modified: branches/home/psmith/restructure/src/io/nio-package.lisp ============================================================================== --- branches/home/psmith/restructure/src/io/nio-package.lisp (original) +++ branches/home/psmith/restructure/src/io/nio-package.lisp Tue Apr 17 00:26:29 2007 @@ -30,7 +30,7 @@ ;; async-fd.lisp async-fd process-read process-write foreign-read-buffer foreign-write-buffer close-sm - recommend-buffer-size close-pending + recommend-buffer-size close-pending process-timeout ;; async-socket.lisp Modified: branches/home/psmith/restructure/src/io/nio-server.lisp ============================================================================== --- branches/home/psmith/restructure/src/io/nio-server.lisp (original) +++ branches/home/psmith/restructure/src/io/nio-server.lisp Tue Apr 17 00:26:29 2007 @@ -46,6 +46,7 @@ ;process reads (handler-case (progn + (process-timeout async-fd) (when (read-ready async-fd) (read-more async-fd)) (when (> (buffer-position (foreign-read-buffer async-fd)) 0) (process-read async-fd)) @@ -63,7 +64,8 @@ (write-more async-fd) (push async-fd removals))) (read-error (re) (push async-fd removals)) - (write-error (we) (push async-fd removals)))) + (write-error (we) (push async-fd removals)) + (timeout (to) (push async-fd removals)))) client-hash) (dolist (async-fd removals) (format-log t "nio-server:process-async-fds processing remove for ~a~%" async-fd) Modified: branches/home/psmith/restructure/src/nio-logger/nio-logger.lisp ============================================================================== --- branches/home/psmith/restructure/src/nio-logger/nio-logger.lisp (original) +++ branches/home/psmith/restructure/src/nio-logger/nio-logger.lisp Tue Apr 17 00:26:29 2007 @@ -63,7 +63,6 @@ (defparameter +log-file-name+ "./out") (defun run-logging-server(listen-ip out-file &optional (allowed-ips "ips.txt")) - (setf nio-yarpc:+process-jobs-inline+ nil) (setf +log-file-name+ out-file) (nio:load-ips allowed-ips) (sb-thread:make-thread #'(lambda()(nio:start-server 'nio-yarpc:yarpc-state-machine :host listen-ip :port 16323 :accept-connection 'nio:check-ip)) :name "nio-server") Modified: branches/home/psmith/restructure/src/protocol/yarpc/nio-yarpc-package.lisp ============================================================================== --- branches/home/psmith/restructure/src/protocol/yarpc/nio-yarpc-package.lisp (original) +++ branches/home/psmith/restructure/src/protocol/yarpc/nio-yarpc-package.lisp Tue Apr 17 00:26:29 2007 @@ -32,9 +32,9 @@ yarpc-state-machine-factory get-packet-factory ;; yarpc-state-machine - yarpc-state-machine job-queue run-job +process-jobs-inline+ +serialise-packet-fn+ + yarpc-state-machine job-queue run-job +process-jobs-inline+ +serialise-packet-fn+ process-timeout ;to be moved - test-rpc test-rpc-list test-rpc-string execute-call defremote + test-rpc test-rpc-list test-rpc-string execute-call defremote process-timeout ;;yarpc-client-state-machine yarpc-client-state-machine remote-execute simulate-connection Modified: branches/home/psmith/restructure/src/protocol/yarpc/yarpc-client-state-machine.lisp ============================================================================== --- branches/home/psmith/restructure/src/protocol/yarpc/yarpc-client-state-machine.lisp (original) +++ branches/home/psmith/restructure/src/protocol/yarpc/yarpc-client-state-machine.lisp Tue Apr 17 00:26:29 2007 @@ -48,11 +48,12 @@ :reader start-time :documentation "The (floating point) start time") (timeout :initarg :timeout - :initform 1.5 + :initform 15 + :reader timeout :documentation "The time in seconds before a timeout should occur, abviously we dont guarantee that this will be honored, it depends on other processing but should be close."))) -(defun remote-job(callback) - (make-instance 'remote-job :callback callback)) +(defun remote-job(callback &key (timeout 15)) + (make-instance 'remote-job :callback callback :timeout timeout)) (defun yarpc-client-state-machine () @@ -67,11 +68,37 @@ (defmethod print-object ((sm yarpc-client-state-machine) stream) (format stream "#" (call-next-method sm nil))) +(defmethod print-object ((job remote-job) stream) + (format stream "#" (start-time job) (timeout job))) + + (defconstant STATE-INITIALISED 0) (defconstant STATE-SENT-REQUEST 1) (defparameter +request-id+ 0) + +(defun check-timeouts(id job) +; (format-log t "Checking timeout on ~A~%" job) + (when (> (get-universal-high-res) (+ (start-time job) (timeout job))) + (format-log t "Timeout detected ~A ~A~%" id job) + t)) + +(defun finish-job (request-id sm result) + "Remove the job from the request map and call the callback with the result" + (let ((remote-job (gethash request-id (request-map sm)))) + (when remote-job + (remhash request-id (request-map sm)) + (funcall (callback remote-job) result)))) + +(defmethod process-timeout((sm yarpc-client-state-machine)) + (let ((requests (request-map sm))) +#+nio-debug (format-log t "yarpc-client-state-machine:process-outgoing-packet called, searching for timeouts in ~A ~%" requests) + (maphash #'(lambda (id job) + (when (check-timeouts id job) (finish-job id sm nil))) + requests))) + + (defmethod process-outgoing-packet((sm yarpc-client-state-machine)) #+nio-debug (format-log t "yarpc-client-state-machine:process-outgoing-packet called, polling the job-queue ~%") (let ((ttd (nio-utils:take (job-queue sm) :blocking-call nil))) @@ -88,8 +115,7 @@ (request-id (request-id response))) #+nio-debug (format-log t "yarpc-client-state-machine:process-incoming-packet :result ~A :request-id ~A~%" result request-id) ; (maphash #'(lambda (k v) (format t "~a -> ~a~%" k v)) (request-map sm)) - (let ((remote-job (gethash request-id (request-map sm)))) - (funcall (callback remote-job) result)))) + (finish-job request-id sm result))) (defparameter *simulate-calls* nil) @@ -107,3 +133,14 @@ (setf (nio:active-conn node) (nio::create-state-machine 'yarpc-client-state-machine 1 1 6)) (push node nio::*nodes-list*))) + + +(defun test-timeout() + (let* ((done nil) + (job (remote-job #'(lambda(x) (format-log t "~A finished~%" x) (setf done t)) :timeout 30))) + (format-log t "Job: ~A~%" job) + (loop while (not done) do + (check-timeouts 99 job) + (format-log t ".~%") + (sleep 1)) + (format-log t "done test~%"))) \ No newline at end of file Modified: branches/home/psmith/restructure/src/protocol/yarpc/yarpc-state-machine.lisp ============================================================================== --- branches/home/psmith/restructure/src/protocol/yarpc/yarpc-state-machine.lisp (original) +++ branches/home/psmith/restructure/src/protocol/yarpc/yarpc-state-machine.lisp Tue Apr 17 00:26:29 2007 @@ -52,12 +52,6 @@ (defconstant STATE-INITIALISED 0) (defconstant STATE-SEND-RESPONSE 1) - -(defparameter +process-jobs-inline+ t - "Set this to make the NIO thread process the RPC calls - warning the procedure should not block!") - - - (defun run-job(&key (blocking t)) #+nio-debug (format-log t "yarpc-state-machine:run-job - Server toplevel waiting for job~%") (let ((server-job (nio-utils:take nio-yarpc:job-queue :blocking-call blocking))) @@ -66,6 +60,7 @@ #+nio-debug (format-log t "yarpc-state-machine:run-job - Server received job ~A~%" job) (nio-utils:add result-queue (list request-id (nio-yarpc:execute-call job))))))) +(defmethod process-timeout((sm yarpc-state-machine))) (defmethod process-outgoing-packet((sm yarpc-state-machine)) #+nio-debug2 (format-log t "yarpc-state-machine:process-outgoing-packet - called, polling the results-queue ~%" ) @@ -78,8 +73,7 @@ ;Process a call method packet by placing it in the job-queue (defmethod process-incoming-packet ((sm yarpc-state-machine) (call call-method-packet)) #+nio-debug (format-log t "yarpc-state-machine:process-incoming-packet - called :sm ~A :packet ~A~%" sm call) - (nio-utils:add job-queue (list (call-string call) (request-id call) (result-queue sm))) - (when +process-jobs-inline+ (run-job :blocking nil))) + (nio-utils:add job-queue (list (call-string call) (request-id call) (result-queue sm)))) From psmith at common-lisp.net Tue Apr 24 03:44:35 2007 From: psmith at common-lisp.net (psmith at common-lisp.net) Date: Mon, 23 Apr 2007 23:44:35 -0400 (EDT) Subject: [nio-cvs] r110 - branches/home/psmith/restructure/src/io Message-ID: <20070424034435.1ABA567096@common-lisp.net> Author: psmith Date: Mon Apr 23 23:44:31 2007 New Revision: 110 Modified: branches/home/psmith/restructure/src/io/nio-package.lisp Log: export remote-host remote-port Modified: branches/home/psmith/restructure/src/io/nio-package.lisp ============================================================================== --- branches/home/psmith/restructure/src/io/nio-package.lisp (original) +++ branches/home/psmith/restructure/src/io/nio-package.lisp Mon Apr 23 23:44:31 2007 @@ -44,5 +44,5 @@ check-ip load-ips ;;nodes - node with-connected-nodes active-conn node-from-socket-repn + node with-connected-nodes active-conn node-from-socket-repn remote-host remote-port )) From psmith at common-lisp.net Thu Apr 26 20:34:43 2007 From: psmith at common-lisp.net (psmith at common-lisp.net) Date: Thu, 26 Apr 2007 16:34:43 -0400 (EDT) Subject: [nio-cvs] r111 - branches/home/psmith/restructure/src/io Message-ID: <20070426203443.19EC4671D1@common-lisp.net> Author: psmith Date: Thu Apr 26 16:34:42 2007 New Revision: 111 Modified: branches/home/psmith/restructure/src/io/ip-authorisation.lisp Log: log allowed ips Modified: branches/home/psmith/restructure/src/io/ip-authorisation.lisp ============================================================================== --- branches/home/psmith/restructure/src/io/ip-authorisation.lisp (original) +++ branches/home/psmith/restructure/src/io/ip-authorisation.lisp Thu Apr 26 16:34:42 2007 @@ -32,7 +32,8 @@ (defun load-ips (filename) (with-open-file (stream filename) - (setf +ip-list+ (read stream)))) + (setf +ip-list+ (read stream))) + (format t "Loaded allowed ips:~A~% " +ip-list+)) (defun check-ip (async-fd) (with-slots (remote-host) (socket async-fd) From psmith at common-lisp.net Thu Apr 26 20:43:14 2007 From: psmith at common-lisp.net (psmith at common-lisp.net) Date: Thu, 26 Apr 2007 16:43:14 -0400 (EDT) Subject: [nio-cvs] r112 - branches/home/psmith/restructure/src/protocol/yarpc Message-ID: <20070426204314.B7C4272085@common-lisp.net> Author: psmith Date: Thu Apr 26 16:43:14 2007 New Revision: 112 Modified: branches/home/psmith/restructure/src/protocol/yarpc/nio-yarpc-package.lisp branches/home/psmith/restructure/src/protocol/yarpc/yarpc-client-state-machine.lisp Log: Allow configurable timeout Modified: branches/home/psmith/restructure/src/protocol/yarpc/nio-yarpc-package.lisp ============================================================================== --- branches/home/psmith/restructure/src/protocol/yarpc/nio-yarpc-package.lisp (original) +++ branches/home/psmith/restructure/src/protocol/yarpc/nio-yarpc-package.lisp Thu Apr 26 16:43:14 2007 @@ -37,5 +37,5 @@ test-rpc test-rpc-list test-rpc-string execute-call defremote process-timeout ;;yarpc-client-state-machine - yarpc-client-state-machine remote-execute simulate-connection + yarpc-client-state-machine remote-execute simulate-connection +rpc-timeout+ )) Modified: branches/home/psmith/restructure/src/protocol/yarpc/yarpc-client-state-machine.lisp ============================================================================== --- branches/home/psmith/restructure/src/protocol/yarpc/yarpc-client-state-machine.lisp (original) +++ branches/home/psmith/restructure/src/protocol/yarpc/yarpc-client-state-machine.lisp Thu Apr 26 16:43:14 2007 @@ -48,11 +48,13 @@ :reader start-time :documentation "The (floating point) start time") (timeout :initarg :timeout - :initform 15 :reader timeout :documentation "The time in seconds before a timeout should occur, abviously we dont guarantee that this will be honored, it depends on other processing but should be close."))) -(defun remote-job(callback &key (timeout 15)) +(defparameter +rpc-timeout+ 60 + "The number of seconds before a remote call is considered timedout") + +(defun remote-job(callback &key (timeout +rpc-timeout+)) (make-instance 'remote-job :callback callback :timeout timeout)) From psmith at common-lisp.net Thu Apr 26 22:33:50 2007 From: psmith at common-lisp.net (psmith at common-lisp.net) Date: Thu, 26 Apr 2007 18:33:50 -0400 (EDT) Subject: [nio-cvs] r113 - branches/home/psmith/restructure/src/io Message-ID: <20070426223350.2B8604E043@common-lisp.net> Author: psmith Date: Thu Apr 26 18:33:49 2007 New Revision: 113 Modified: branches/home/psmith/restructure/src/io/ip-authorisation.lisp branches/home/psmith/restructure/src/io/nio-package.lisp Log: Added allow-ips to ip-authorisation Modified: branches/home/psmith/restructure/src/io/ip-authorisation.lisp ============================================================================== --- branches/home/psmith/restructure/src/io/ip-authorisation.lisp (original) +++ branches/home/psmith/restructure/src/io/ip-authorisation.lisp Thu Apr 26 18:33:49 2007 @@ -35,8 +35,18 @@ (setf +ip-list+ (read stream))) (format t "Loaded allowed ips:~A~% " +ip-list+)) +;(allow-ips '("127.0.0.1")) +(defun allow-ips (ip-list) + "Add each ip in the allow ips list to our list" + (dolist (item ip-list) + (when (not (check-ip-str item)) (push item +ip-list+)))) + + +(defun check-ip-str(ip-string) + (member ip-string +ip-list+ :test 'string-equal)) + (defun check-ip (async-fd) (with-slots (remote-host) (socket async-fd) (let ((str-rep (format nil "~{~a~^.~}" (reverse remote-host)))) (format t "ip-authorisation:check-ip ~A ~A~%" str-rep +ip-list+) - (member str-rep +ip-list+ :test 'string-equal)))) \ No newline at end of file + (check-ip-str str-rep)))) Modified: branches/home/psmith/restructure/src/io/nio-package.lisp ============================================================================== --- branches/home/psmith/restructure/src/io/nio-package.lisp (original) +++ branches/home/psmith/restructure/src/io/nio-package.lisp Thu Apr 26 18:33:49 2007 @@ -41,7 +41,7 @@ packet write-bytes get-packet-size ;;ip-authorisation - check-ip load-ips + check-ip load-ips allow-ips ;;nodes node with-connected-nodes active-conn node-from-socket-repn remote-host remote-port From psmith at common-lisp.net Fri Apr 27 18:44:48 2007 From: psmith at common-lisp.net (psmith at common-lisp.net) Date: Fri, 27 Apr 2007 14:44:48 -0400 (EDT) Subject: [nio-cvs] r114 - branches/home/psmith/restructure/src/io Message-ID: <20070427184448.34C9E671A6@common-lisp.net> Author: psmith Date: Fri Apr 27 14:44:47 2007 New Revision: 114 Modified: branches/home/psmith/restructure/src/io/nodes.lisp Log: Added connected nodes count io/nio-package.lisp Modified: branches/home/psmith/restructure/src/io/nodes.lisp ============================================================================== --- branches/home/psmith/restructure/src/io/nodes.lisp (original) +++ branches/home/psmith/restructure/src/io/nodes.lisp Fri Apr 27 14:44:47 2007 @@ -98,3 +98,10 @@ (defmacro with-connected-nodes ((node) &rest body) `(dolist (,node *nodes-list*) (when (active-conn ,node) , at body))) + +(defun connected-nodes-count() + (let ((count 0)) + (with-connected-nodes (node) + (incf count)) + count)) + \ No newline at end of file From psmith at common-lisp.net Fri Apr 27 22:39:46 2007 From: psmith at common-lisp.net (psmith at common-lisp.net) Date: Fri, 27 Apr 2007 18:39:46 -0400 (EDT) Subject: [nio-cvs] r115 - branches/home/psmith/restructure/src/io Message-ID: <20070427223946.D7F0A7B143@common-lisp.net> Author: psmith Date: Fri Apr 27 18:39:46 2007 New Revision: 115 Modified: branches/home/psmith/restructure/src/io/nio-package.lisp Log: export connected-nodes-count Modified: branches/home/psmith/restructure/src/io/nio-package.lisp ============================================================================== --- branches/home/psmith/restructure/src/io/nio-package.lisp (original) +++ branches/home/psmith/restructure/src/io/nio-package.lisp Fri Apr 27 18:39:46 2007 @@ -44,5 +44,5 @@ check-ip load-ips allow-ips ;;nodes - node with-connected-nodes active-conn node-from-socket-repn remote-host remote-port + node with-connected-nodes active-conn node-from-socket-repn remote-host remote-port connected-nodes-count ))