From 8e5ea8261110e97a8df56cfc9c83028081d902fb Mon Sep 17 00:00:00 2001 From: David Fifield Date: Thu, 20 Apr 2023 09:47:56 -0400 Subject: [PATCH] Add a scanner error check to ClusterCounter.Count. It was silently exiting at the "recordingStart":"2022-09-23T17:06:59.680537075Z" line, the first line whose length (66873) exceeds bufio.MaxScanTokenSize. Now distinctcounter exits with an error status instead of reporting partial results. $ ./distinctcounter -from 2023-01-01T00:00:00Z -to 2023-01-10T00:00:00Z -in metrics-ip-salted.jsonl 2023/04/20 13:54:11 unable to count:bufio.Scanner: token too long --- common/ipsetsink/sinkcluster/reader.go | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/common/ipsetsink/sinkcluster/reader.go b/common/ipsetsink/sinkcluster/reader.go index 3f7a08f..78d383b 100644 --- a/common/ipsetsink/sinkcluster/reader.go +++ b/common/ipsetsink/sinkcluster/reader.go @@ -55,6 +55,10 @@ func (c ClusterCounter) Count(reader io.Reader) (*ClusterCountResult, error) { return nil, err } } + err = inputScanner.Err() + if err != nil { + return nil, err + } result.Sum = counter.Count() return &result, nil }