Update package

Author: phaag

README.md

@@ -1,5 +1,7 @@
 # go-nfdump
 
+[![Go Reference](https://pkg.go.dev/badge/github.com/phaag/go-nfdump.svg)](https://pkg.go.dev/github.com/phaag/go-nfdump)
+
 This Go module allows to read and process files created by [nfdump](https://github.com/phaag/nfdump), the netflow/ipfix/sflow collector and processing tools.
 
 This module is experimental and does not yet decode all available nfdump record extensions. It reads and processes only nfdump v2 files, which are created by nfdump-1.7.x. Files created with nfdump-1.6.x are recogized but skipped for decoding.
@@ -15,9 +17,9 @@ package main
 import (
 	"flag"
 	"fmt"
-	"github.com/phaag/go-nfdump/nffile"
-	"github.com/phaag/go-nfdump/nfrecord"
 	"os"
+
+	nfdump "github.com/phaag/go-nfdump"
 )
 
 var (
@@ -30,28 +32,31 @@ func main() {
 		fmt.Fprintf(os.Stderr, "Usage of %s [flags]\n", os.Args[0])
 		flag.PrintDefaults()
 	}
-	
+
 	flag.Parse()
-	
+
 	if len(*fileName) == 0 {
 		fmt.Printf("Filename required\n")
 		flag.PrintDefaults()
 		os.Exit(255)
 	}
-	
-	nffile := nffile.New()
+
+	nffile := nfdump.New()
+
 	if err := nffile.Open(*fileName); err != nil {
 		fmt.Printf("Failed to open nf file: %v\n", err)
 		os.Exit(255)
 	}
-	
+
 	// print nffile stats
 	fmt.Printf("nffile:\n%v", nffile)
-	
+
 	// Dump flow records
-	recordChannel, _ := nfrecord.AllRecords(nffile)
+	recordChannel, _ := nfdump.AllRecords(nffile)
+	cnt := 0
 	for record := range recordChannel {
-		fmt.Printf("%v\n", record)
+		cnt++
+		fmt.Printf("record: %d\n%v\n", cnt, record)
 		genericFlow := record.GenericFlow()
 		if genericFlow != nil {
 			fmt.Printf("SrcPort: %d\n", genericFlow.SrcPort)
@@ -71,10 +76,9 @@ func main() {
 		*/
 	}
 }
-
 ```
 
-The `nfrecord/defs.go` file includes nfdump's `nfxV3.h` header file to convert individual record extensions into appropriate Golang records. So far the generic, misc, flowCount, vlan and asRouting extensions as well as IPv4/IPv6 addresses are available through the interface. See the nfxV3.go file for its definitions.
+The `defs.go` file includes nfdump's `nfxV3.h` header file to convert individual record extensions into appropriate Golang records. So far the generic, misc, flowCount, vlan and asRouting extensions as well as IPv4/IPv6 addresses are available through the interface. See the nfxV3.go file for its definitions.
 
 If you modify the `defs.go` file, generate `nfxV3.go` use the go command
 

nfrecord/_obj/_cgo_.o renamed to _obj/_cgo_.o

@@ -0,0 +1,59 @@
+// Copyright © 2023 Peter Haag [email protected]
+// All rights reserved.
+//
+// Use of this source code is governed by the license that can be
+// found in the LICENSE file.
+
+package nfdump
+
+import (
+	"bytes"
+	"compress/bzip2"
+	"fmt"
+	"io"
+
+	"github.com/pierrec/lz4/v4"
+	lzo "github.com/rasky/go-lzo"
+)
+
+func (nfFile *NfFile) uncompressBlock(blockHeader *DataBlockHeader) ([]byte, error) {
+
+	dataBlock := make([]byte, blockHeader.Size)
+	if _, err := io.ReadAtLeast(nfFile.file, dataBlock, int(blockHeader.Size)); err != nil {
+		return nil, fmt.Errorf("nfFile read appendix data block: %v", err)
+	}
+
+	switch nfFile.Header.Compression {
+	case NOT_COMPRESSED:
+	case LZO_COMPRESSED:
+		out, err := lzo.Decompress1X(bytes.NewReader(dataBlock), int(blockHeader.Size), 2*int(blockHeader.Size))
+		if err != nil {
+			return nil, fmt.Errorf("nfFile uncompress lzo1x-1 data block: %v", err)
+		}
+		dataBlock = out
+		blockHeader.Size = uint32(len(out))
+	case BZ2_COMPRESSED:
+		reader := bzip2.NewReader(bytes.NewReader(dataBlock))
+		out := make([]byte, 3*len(dataBlock))
+		n, err := reader.Read(out)
+		if err != nil {
+			return nil, fmt.Errorf("nfFile uncompress bzip2 data block: %v", err)
+		}
+		out = out[:n]
+		dataBlock = out
+		blockHeader.Size = uint32(n)
+	case LZ4_COMPRESSED:
+		out := make([]byte, 3*len(dataBlock))
+		n, err := lz4.UncompressBlock(dataBlock, out)
+		if err != nil {
+			return nil, fmt.Errorf("nfFile uncompress lz4 data block: %v", err)
+		}
+		out = out[:n]
+		dataBlock = out
+		blockHeader.Size = uint32(n)
+	default:
+		return nil, fmt.Errorf("unknown data block compression: %d", nfFile.Header.Compression)
+	}
+
+	return dataBlock, nil
+}

compress.go

@@ -0,0 +1,41 @@
+//go:build ignore
+
+// Copyright © 2023 Peter Haag [email protected]
+// All rights reserved.
+//
+// Use of this source code is governed by the license that can be
+// found in the LICENSE file.
+
+package nfdump
+
+//#define GOLANG 1
+//#include <stdint.h>
+//#include "nfxV3.h"
+//#include "id.h"
+//
+import "C"
+
+const EXnull = uint(C.EXnull)
+const EXgenericFlowID = uint16(C.EXgenericFlowID)
+const EXipv4FlowID = uint16(C.EXipv4FlowID)
+const EXipv6FlowID = uint16(C.EXipv6FlowID)
+const EXflowMiscID = uint16(C.EXflowMiscID)
+const EXcntFlowID = uint16(C.EXcntFlowID)
+const EXvLanID = uint16(C.EXvLanID)
+const EXasRoutingID = uint16(C.EXasRoutingID)
+
+const V3_FLAG_EVENT = uint(C.V3_FLAG_EVENT)
+const V3_FLAG_SAMPLED = uint(C.V3_FLAG_SAMPLED)
+const V3_FLAG_ANON = uint(C.V3_FLAG_ANON)
+
+const V3Record = uint16(C.V3Record)
+
+const MAXEXTENSIONS = uint16(C.MAXEXTENSIONS)
+
+type recordHeaderV3 C.struct_recordHeaderV3_s
+
+type EXgenericFlow C.struct_EXgenericFlow_s
+type EXflowMisc C.struct_EXflowMisc_s
+type EXcntFlow C.struct_EXcntFlow_s
+type EXvLan C.struct_EXvLan_s
+type EXasRouting C.struct_EXasRouting_s

defs.go

@@ -0,0 +1,70 @@
+// Copyright © 2023 Peter Haag [email protected]
+// All rights reserved.
+//
+// Use of this source code is governed by the license that can be
+// found in the LICENSE file.
+
+package main
+
+import (
+	"flag"
+	"fmt"
+	"os"
+
+	nfdump "github.com/phaag/go-nfdump"
+)
+
+var (
+	fileName = flag.String("r", "", "nfdump file to read")
+)
+
+func main() {
+
+	flag.CommandLine.Usage = func() {
+		fmt.Fprintf(os.Stderr, "Usage of %s [flags]\n", os.Args[0])
+		flag.PrintDefaults()
+	}
+
+	flag.Parse()
+
+	if len(*fileName) == 0 {
+		fmt.Printf("Filename required\n")
+		flag.PrintDefaults()
+		os.Exit(255)
+	}
+
+	nffile := nfdump.New()
+
+	if err := nffile.Open(*fileName); err != nil {
+		fmt.Printf("Failed to open nf file: %v\n", err)
+		os.Exit(255)
+	}
+
+	// print nffile stats
+	fmt.Printf("nffile:\n%v", nffile)
+
+	// Dump flow records
+	recordChannel, _ := nfdump.AllRecords(nffile)
+	cnt := 0
+	for record := range recordChannel {
+		cnt++
+		fmt.Printf("record: %d\n%v\n", cnt, record)
+		genericFlow := record.GenericFlow()
+		if genericFlow != nil {
+			fmt.Printf("SrcPort: %d\n", genericFlow.SrcPort)
+			fmt.Printf("DstPort: %d\n", genericFlow.DstPort)
+		}
+		ipAddr := record.IP()
+		if ipAddr != nil {
+			fmt.Printf("SrcIP: %v\n", ipAddr.SrcIP)
+			fmt.Printf("DstIP: %v\n", ipAddr.DstIP)
+		}
+		/*
+			other extension
+			flowMisc := record.FlowMisc()
+			cntFlow := record.CntFlow()
+			vLan := record.VLan()
+			asRouting := record.AsRouting()
+		*/
+	}
+}

example/reader/main.go

@@ -1,33 +1,11 @@
-/*
- *  Copyright (c) 2023, Peter Haag
- *  All rights reserved.
- *
- *  Redistribution and use in source and binary forms, with or without
- *  modification, are permitted provided that the following conditions are met:
- *
- *   * Redistributions of source code must retain the above copyright notice,
- *     this list of conditions and the following disclaimer.
- *   * Redistributions in binary form must reproduce the above copyright notice,
- *     this list of conditions and the following disclaimer in the documentation
- *     and/or other materials provided with the distribution.
- *   * Neither the name of the author nor the names of its contributors may be
- *     used to endorse or promote products derived from this software without
- *     specific prior written permission.
- *
- *  THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
- *  AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
- *  IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
- *  ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
- *  LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
- *  CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
- *  SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
- *  INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
- *  CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
- *  ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
- *  POSSIBILITY OF SUCH DAMAGE.
- */
+// Copyright © 2023 Peter Haag [email protected]
+// All rights reserved.
+//
+// Use of this source code is governed by the license that can be
+// found in the LICENSE file.
 
-package nffile
+// Package nfdump provides an API for nfdump files
+package nfdump
 
 import (
 	"bytes"