[Small-cl-src] Netflow log reader (for the flowd logger from www.mindrot.org)
Ingvar
ingvar at cathouse.bofh.se
Wed Oct 20 18:15:37 UTC 2004
;;; This is (C) Ingvar Mattsson, 2004
;;; <ingvar at hexapodia.net>
;;; This code uses the file format specification outlined in store.h
;;; in the netflow logger daemon downloadable from
;;; http://www.mindrot.org/flowd.html
;;;
;;; This code is available under the BSD license, please preserve the
;;; relevant copyright notices.
;;;
;;; The store.h file is:
;;; /*
;;; * Copyright (c) 2004 Damien Miller <djm at mindrot.org>
;;; *
;;; * Permission to use, copy, modify, and distribute this software for any
;;; * purpose with or without fee is hereby granted, provided that the above
;;; * copyright notice and this permission notice appear in all copies.
;;; *
;;; * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
;;; * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
;;; * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
;;; * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
;;; * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
;;; * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
;;; * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
;;; */
(defpackage #:flowd
(:use #:cl)
(:shadow #:tag #:stream)
(:export
#:+store-magic+ #:+store-version+ #:store-field-tag #:store-field-recv-time
#:store-field-proto-flags-tos #:store-field-agent-addr4
#:store-field-agent-addr6 #:store-field-src-addr4 #:store-field-src-addr6
#:store-field-dst-addr4 #:store-field-dst-addr6 #:store-field-gateway-addr4
#:store-field-gateway-addr6 #:store-field-srcdst-port #:store-field-packets
#:store-field-octets #:store-field-if-indices #:store-field-agent-info
#:store-field-flow-times #:store-field-as-info
#:store-field-flow-engine-info #:store-field-crc32 #:store-field-all
#:open-log #:read-flow #:close-log #:fields #:tag #:recv-time
#:proto-flags-tos #:agent-addr #:src-addr #:dst-addr #:gateway-addr
#:src-port #:dst-port #:packets #:if-index-in #:if-index-out
#:sys-uptime-ms #:time-sec #:time-nanosec #:netflow-version
#:flow-start #:flow-finish #:src-as #:dst-as #:src-mask #:dst-mask
#:engine-type #:engine-id #:flow-sequence #:src-net #:dst-net #:octets
#:packets #:start-time #:format-ipv4))
(in-package #:flowd)
(defvar *ipv4-netmasks*
(apply #'vector
(loop for n from 0 to 32
for mask = 0 then (logior #x80000000 (ash mask -1))
collect mask)))
(defconstant +store-magic+ #x012cf047)
(defconstant +store-version+ 2)
(defconstant +store-field-tag+ (ash 1 0))
(defconstant +store-field-recv-time+ (ash 1 1))
(defconstant +store-field-proto-flags-tos+ (ash 1 2))
(defconstant +store-field-agent-addr4+ (ash 1 3))
(defconstant +store-field-agent-addr6+ (ash 1 4))
(defconstant +store-field-src-addr4+ (ash 1 5))
(defconstant +store-field-src-addr6+ (ash 1 6))
(defconstant +store-field-dst-addr4+ (ash 1 7))
(defconstant +store-field-dst-addr6+ (ash 1 8))
(defconstant +store-field-gateway-addr4+ (ash 1 9))
(defconstant +store-field-gateway-addr6+ (ash 1 10))
(defconstant +store-field-srcdst-port+ (ash 1 11))
(defconstant +store-field-packets+ (ash 1 12))
(defconstant +store-field-octets+ (ash 1 13))
(defconstant +store-field-if-indices+ (ash 1 14))
(defconstant +store-field-agent-info+ (ash 1 15))
(defconstant +store-field-flow-times+ (ash 1 16))
(defconstant +store-field-as-info+ (ash 1 17))
(defconstant +store-field-flow-engine-info+ (ash 1 18))
(defconstant +store-field-crc32+ (ash 1 30))
(defconstant +store-field-all+ (1- (ash 1 19)))
(defclass store-header ()
((magic :reader magic :initarg :magic)
(version :reader version :initarg :version)
(start-time :reader start-time :initarg :start-time)
(flags :reader flags :initarg :flags)
(stream :reader stream :initarg :stream)
))
(defclass flow ()
((fields :accessor fields :initarg :fields :initform nil)
(tag :accessor tag :initarg :tag :initform nil)
(recv-time :accessor recv-time :initarg :recv-time :initform nil)
(tcp-flags :accessor tcp-flags :initarg :tcp-flags :initform nil)
(protocol :accessor protocol :initarg :protocol :initform nil)
(tos :accessor tos :initarg :tos :initform nil)
(agent-addr :accessor agent-addr :initarg :agent-addr :initform nil)
(src-addr :accessor src-addr :initarg :src-addr :initform nil)
(dst-addr :accessor dst-addr :initarg :dst-addr :initform nil)
(gateway-addr :accessor gateway-addr :initarg :gateway-addr :initform nil)
(src-port :accessor src-port :initarg :src-port :initform nil)
(dst-port :accessor dst-port :initarg :dst-port :initform nil)
(packets :accessor packets :initarg :packets :initform nil)
(octets :accessor octets :initarg :octets :initform nil)
(if-index-in :accessor if-index-in :initarg :if-index-in :initform nil)
(if-index-out :accessor if-index-out :initarg :if-index-out :initform nil)
(sys-uptime-ms :accessor sys-uptime-ms :initarg :sys-uptime-ms :initform nil)
(time-sec :accessor time-sec :initarg :time-sec :initform nil)
(time-nanosec :accessor time-nanosec :initarg :time-nanosec :initform nil)
(netflow-version :accessor netflow-version :initarg :netflow-version :initform nil)
(flow-start :accessor flow-start :initarg :flow-start :initform nil)
(flow-finish :accessor flow-finish :initarg :flow-finish :initform nil)
(src-as :accessor src-as :initarg :src-as :initform nil)
(dst-as :accessor dst-as :initarg :dst-as :initform nil)
(src-mask :accessor src-mask :initarg :src-mask :initform nil)
(dst-mask :accessor dst-mask :initarg :dst-mask :initform nil)
(engine-type :accessor engine-type :initarg :engine-type :initform nil)
(engine-id :accessor engine-id :initarg :engine-id :initform nil)
(flow-sequence :accessor flow-sequence :initarg :flow-sequence :initform nil)
))
(defclass ipaddr ()
((address :accessor address :initarg :address)))
(defclass ipv4 (ipaddr) ())
(defclass ipv6 (ipaddr) ())
(defun make-ipv4 (addr)
"This function is currently a no-op"
;;(make-instance 'ipv4 :address addr)
(identity addr)
)
(defun make-ipv6 (addr)
"This function is currently a no-op"
;;(make-instance 'ipv6 :address addr)
(identity addr)
)
(defmacro when-flagged (flag &body body)
"Checks if a given flag is set. The flag field is expected to be named
FIELDS and is for use inside READ-FLOW only!"
`(when (not (zerop (logand fields ,flag)))
, at body))
(defun read-n-bytes (stream n)
"Read from STREAM a total of N bytes, mung them together as a single
integer. Expects 8-bit bytes."
(let ((acc 0))
(loop for r from 1 to n
do (setf acc (logior (ash acc 8) (read-byte stream))))
acc))
(defun read-flow (flow-header &optional flow-obj)
"(read-flow <flow-header> &optional flow-object)
This function reads one flow entry from a log file (return value from
OPEN-LOG) and returns it. If a flow object is passed in as an optional
parameter, this flow object is re-used for storage instead of allocating
a new instance."
(let ((stream (stream flow-header)))
(let ((fields (read-n-bytes stream 4)))
(let ((flow (if flow-obj
(progn
(setf (fields flow-obj) fields)
flow-obj)
(make-instance 'flow :fields fields)))
pad)
(when-flagged +store-field-tag+
(setf (tag flow) (read-n-bytes stream 4)))
(when-flagged +store-field-recv-time+
(setf (recv-time flow) (read-n-bytes stream 4)))
(when-flagged +store-field-proto-flags-tos+
(setf (tcp-flags flow) (read-n-bytes stream 1))
(setf (protocol flow) (read-n-bytes stream 1))
(setf (tos flow) (read-n-bytes stream 1))
(setf pad (read-n-bytes stream 1)))
(when-flagged +store-field-agent-addr4+
(setf (agent-addr flow) (make-ipv4 (read-n-bytes stream 4))))
(when-flagged +store-field-agent-addr6+
(setf (agent-addr flow) (make-ipv6 (read-n-bytes stream 16))))
(when-flagged +store-field-src-addr4+
(setf (src-addr flow) (make-ipv4 (read-n-bytes stream 4))))
(when-flagged +store-field-src-addr6+
(setf (src-addr flow) (make-ipv6 (read-n-bytes stream 16))))
(when-flagged +store-field-dst-addr4+
(setf (dst-addr flow) (make-ipv4 (read-n-bytes stream 4))))
(when-flagged +store-field-dst-addr6+
(setf (dst-addr flow) (make-ipv6 (read-n-bytes stream 16))))
(when-flagged +store-field-gateway-addr4+
(setf (gateway-addr flow) (make-ipv4 (read-n-bytes stream 4))))
(when-flagged +store-field-gateway-addr6+
(setf (gateway-addr flow) (make-ipv6 (read-n-bytes stream 16))))
(when-flagged +store-field-srcdst-port+
(setf (src-port flow) (read-n-bytes stream 2))
(setf (dst-port flow) (read-n-bytes stream 2)))
(when-flagged +store-field-packets+
(setf (packets flow) (read-n-bytes stream 8)))
(when-flagged +store-field-octets+
(setf (octets flow) (read-n-bytes stream 8)))
(when-flagged +store-field-if-indices+
(setf (if-index-in flow) (read-n-bytes stream 2))
(setf (if-index-out flow) (read-n-bytes stream 2)))
(when-flagged +store-field-agent-info+
(setf (sys-uptime-ms flow) (read-n-bytes stream 4))
(setf (time-sec flow) (read-n-bytes stream 4))
(setf (time-nanosec flow) (read-n-bytes stream 4))
(setf (netflow-version flow) (read-n-bytes stream 2))
(setf pad (read-n-bytes stream 2)))
(when-flagged +store-field-flow-times+
(setf (flow-start flow) (read-n-bytes stream 4))
(setf (flow-finish flow) (read-n-bytes stream 4)))
(when-flagged +store-field-as-info+
(setf (src-as flow) (read-n-bytes stream 2))
(setf (dst-as flow) (read-n-bytes stream 2))
(setf (src-mask flow) (read-n-bytes stream 1))
(setf (dst-mask flow) (read-n-bytes stream 1))
(setf pad (read-n-bytes stream 2)))
(when-flagged +store-field-flow-engine-info+
(setf (engine-type flow) (read-n-bytes stream 1))
(setf (engine-id flow) (read-n-bytes stream 1))
(setf pad (read-n-bytes stream 2))
(setf (flow-sequence flow) (read-n-bytes stream 4)))
(when-flagged +store-field-crc32+
(setf pad (read-n-bytes stream 4)))
flow))))
(defun open-log (file)
"(open-log <file name>
This function opens a new log file and returns a header structure containing
the relevant file header information."
(let ((stream (open file :element-type '(unsigned-byte 8) :direction :input)))
(let ((magic (read-n-bytes stream 4)))
(let ((version (read-n-bytes stream 4)))
(let ((start-time (read-n-bytes stream 4)))
(let ((flags (read-n-bytes stream 4)))
(make-instance 'store-header
:magic magic
:version version
:start-time start-time
:flags flags
:stream stream)))))))
(defun close-log (flow)
"(close-log flow)
This function closes the log file associated with a storage header."
(close (stream flow)))
(defmacro formatted-addr (flow-obj slot)
(let ((flags
(case slot
(dst-addr (list +store-field-dst-addr4+ +store-field-dst-addr6+))
(src-addr (list +store-field-src-addr4+ +store-field-src-addr6+))
(gateway-addr (list +store-field-gateway-addr4+ +store-field-gateway-addr6+))
(agent-addr (list +store-field-agent-addr4+ +store-field-agent-addr6+))))
(flow flow-obj))
`(format-addr ,flow ',flags (,slot ,flow))))
(defun format-addr (flow-obj flags chunk)
(let ((flag4 (car flags))
(flag6 (cadr flags))
(fields (fields flow-obj)))
(or (when (= flag4 (logand flag4 fields))
(format-ipv4 chunk))
(when (= flag6 (logand flag6 fields))
(format nil "~X" chunk)))))
(defun format-ipv4 (chunk &optional stream mask)
"(format-ipv4 binary-chunk &optiona stream mask)
This function outputs an IPv4 address as a dotted quad to STREAM. If a
netmask is passed in, it's outputted with the dotted quad in CIDR notation."
(if mask (format stream "~D.~D.~D.~D/~D"
(ldb (byte 8 24) chunk) (ldb (byte 8 16) chunk)
(ldb (byte 8 8) chunk) (ldb (byte 8 0) chunk) mask)
(format stream "~D.~D.~D.~D"
(ldb (byte 8 24) chunk) (ldb (byte 8 16) chunk)
(ldb (byte 8 8) chunk) (ldb (byte 8 0) chunk))))
(defun src-net (flow-obj &optional (stream nil) (formatted nil))
"(src-net flow-obj &optional stream formatted-p)
This function extracts the source network and masks it against the
relevant IPv4 netmask and returns the network part. If given a STREAM and
FORMATTED-P is not null, the resulting netblock is emitted using
FORMAT-IPV4 to the indicated stream."
(let ((bitmask (logior +store-field-as-info+ +store-field-src-addr4+)))
(when (= bitmask (logand (fields flow-obj) bitmask))
(let ((masklen (src-mask flow-obj)))
(let ((netmask (aref *ipv4-netmasks* masklen)))
(let ((netblock (logand (src-addr flow-obj) netmask)))
(if formatted
(format-ipv4 netblock stream masklen)
netblock)))))))
(defun dst-net (flow-obj &optional (stream nil) (formatted nil))
"(dst-net flow-obj &optional stream formatted-p)
This function extracts the destination network and masks it against the
relevant IPv4 netmask and returns the network part. If given a STREAM and
FORMATTED-P is not null, the resulting netblock is emitted using
FORMAT-IPV4 to the indicated stream."
(let ((bitmask (logior +store-field-as-info+ +store-field-dst-addr4+)))
(when (= bitmask (logand (fields flow-obj) bitmask))
(let ((masklen (dst-mask flow-obj)))
(let ((netmask (aref *ipv4-netmasks* masklen)))
(let ((netblock (logand (dst-addr flow-obj) netmask)))
(if formatted
(format-ipv4 netblock stream masklen)
netblock)))))))
--
//Ingvar
More information about the Small-cl-src
mailing list