Initial release

LICENSE

@@ -0,0 +1,24 @@
+Copyright (c) 2020 Jason T. Lenz. All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are
+met:
+
+   * Redistributions of source code must retain the above copyright
+notice, this list of conditions and the following disclaimer.
+   * Redistributions in binary form must reproduce the above
+copyright notice, this list of conditions and the following disclaimer
+in the documentation and/or other materials provided with the
+distribution.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

Makefile

@@ -0,0 +1,71 @@
+.POSIX:
+
+PNAME = chunkio
+
+RTEMPLATE ?= ../repo-template
+
+all: doc mkFile
+
+doc: docMain
+
+cleanDoc: cleanDocMain
+
+.DEFAULT_GOAL := all
+
+.PHONY: all doc cleanDoc
+
+#---Golang Library Section---
+
+GO ?= go
+GOFLAGS ?=
+GOSRC != find . -name '*.go'
+
+check: $(GOSRC)
+	$(GO) test $(GOFLAGS)
+
+.PHONY: check
+
+#---Generate Main Documents---
+
+DOCMAIN := README.md LICENSE
+
+README.md: template/README.md.got
+	pgot -i ":$(RTEMPLATE)" -o $@ $<
+
+LICENSE: $(RTEMPLATE)/LICENSE.src/BSD-2-clause.got
+	pgot -i ":$(RTEMPLATE)" -o $@ $<
+
+docMain: $(DOCMAIN)
+
+cleanDocMain:
+	$(RM) $(DOCMAIN)
+
+.PHONY: docMain, cleanDocMain
+
+#---Generate Makefile---
+
+Makefile: template/Makefile.got
+	pgot -i ":$(RTEMPLATE)" -o $@ $<
+
+mkFile: Makefile
+
+regenMkFile:
+	pgot -i ":$(RTEMPLATE)" -o Makefile template/Makefile.got
+
+.PHONY: mkFile regenMkFile
+
+#---Lint Helper Target---
+
+lint:
+	@find . -path ./.git -prune -or \
+		-type f -and -not -name 'Makefile' \
+		-exec grep -Hn '<no value>' '{}' ';'
+
+#---TODO Helper Target---
+
+todo:
+	@find . -path ./.git -prune -or \
+		-type f -and -not -name 'Makefile' \
+		-exec grep -Hn TODO '{}' ';'
+
+# vim:set noet tw=80:

README.md

@@ -0,0 +1,103 @@
+# chunkio
+
+**chunkio** is a golang package that provides functionality for transparently
+reading a subset of a stream containing a user defined ending byte sequence.
+When the byte sequence is reached an EOF is returned. This sub stream can be
+accessed or passed to other routines as standard Reader objects.
+
+## Interface
+
+###Variables
+
+```text
+var ErrInvalidKey = errors.New("chunkio: invalid key definition")
+```
+
+###Types
+
+```text
+type Reader struct {
+    // Has unexported fields.
+}
+    Reader implements chunkio functionality wrapped around an io.Reader object
+
+func NewReader(rd io.Reader) *Reader
+    NewReader creates a new chunk reader
+
+func (c *Reader) GetErr() error
+    GetErr returns the error status for the current active chunkio stream
+
+func (c *Reader) GetKey() []byte
+    GetKey returns the key for the current active chunkio stream
+
+func (c *Reader) Read(p []byte) (int, error)
+    Read implements the standard Reader interface allowing chunkio to be used
+    anywhere a standard Reader can be used. Read reads data into p. It returns
+    the number of bytes read into p. The bytes are taken from at most one Read
+    on the underlying Reader, hence n may be less than len(p). When the key is
+    reached (EOF for the stream chunk), the count will be zero and err will be
+    io.EOF. If the key has been set to nil, the Read function performs exactly
+    like the underlying stream Read function (no key scanning).
+
+func (c *Reader) Reset()
+    Reset puts the chunkio stream back into a readable state. This can be used
+    when the end of a chunk is reached to enable reading the next chunk.
+
+func (c *Reader) SetKey(key []byte) error
+    SetKey updates the search key. The search key can also be cleared by
+    providing a nil key.
+```
+
+## Example usage.
+
+```go
+import (
+        "bytes"
+        "fmt"
+        "io/ioutil"
+        "git.lenzplace.org/lenzj/chunkio"
+        "strings"
+)
+
+func ExampleUppercase() {
+        example := []byte("the quick {U}brown fox jumps{R} over the lazy dog")
+        cio := chunkio.NewReader(bytes.NewReader(example))
+        cio.SetKey([]byte("{U}"))
+        s1, _ := ioutil.ReadAll(cio)
+        cio.Reset()
+        cio.SetKey([]byte("{R}"))
+        s2, _ := ioutil.ReadAll(cio)
+        cio.Reset()
+        s3, _ := ioutil.ReadAll(cio)
+        fmt.Print(string(s1)+strings.ToUpper(string(s2))+string(s3))
+        // Output: the quick BROWN FOX JUMPS over the lazy dog
+}
+```
+
+## Running the tests
+
+```
+$ make check
+```
+
+## Contributing
+
+If you have a bugfix, update, issue or feature enhancement the best way to reach
+me is by following the instructions in the link below.  Thank you!
+
+<https://blog.lenzplace.org/about/contact.html>
+
+
+## Versioning
+
+I follow the [SemVer](http://semver.org/) strategy for versioning. The latest
+version is listed in the [releases](/lenzj/chunkio/releases) section. 
+
+
+## License
+
+This project is licensed under a BSD two clause license - see the
+[LICENSE](LICENSE) file for details.
+
+
+<!-- vim:set ts=4 sw=4 et tw=80: -->

chunkio.go

@@ -0,0 +1,183 @@
+// Copyright (c) 2019 Jason T. Lenz.  All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be found
+// in the LICENSE file.
+
+/*
+Package chunkio provides functionality for transparently reading a subset of a
+stream containing a user defined ending byte sequence.  When the byte sequence
+is reached an EOF is returned.  This sub stream can be accessed or passed to
+other routines as standard Reader objects.
+*/
+package chunkio
+
+import (
+	"bytes"
+	"errors"
+	"io"
+)
+
+const (
+	minKeyLength = 1
+	bufAdd       = 4096 // buffAdd plus key length = buffer size
+)
+
+var ErrInvalidKey = errors.New("chunkio: invalid key definition")
+
+// Reader implements chunkio functionality wrapped around an io.Reader object
+type Reader struct {
+	rd      io.Reader    // Underlying Reader
+	key     []byte       // key that delineates end of chunk
+	buf     bytes.Buffer // A buffer to provide "read ahead" ability
+	bufSize int          // The target buffer size
+	err     error        // Current error state of chunkio Reader
+	ierr    error        // Current error state of underlying Reader
+	scan    int          // Number of bytes in buffer that have already been scanned for key
+	found   bool         // True if key exists in buffer. Position is in scan in that case
+}
+
+// NewReader creates a new chunk reader.
+func NewReader(rd io.Reader) *Reader {
+	return &Reader{
+		rd:      rd,
+		key:     nil,
+		buf:     bytes.Buffer{},
+		bufSize: 0,
+		err:     nil,
+		ierr:    nil,
+		scan:    0,
+		found:   false,
+	}
+}
+
+// GetKey returns the key for the current active chunkio stream.
+func (c *Reader) GetKey() []byte {
+	return c.key
+}
+
+// GetErr returns the error status for the current active chunkio stream.
+func (c *Reader) GetErr() error {
+	return c.err
+}
+
+// SetKey updates the search key.  The search key can also be cleared by
+// providing a nil key.
+func (c *Reader) SetKey(key []byte) error {
+	if key == nil {
+		c.key = key
+		return nil
+	}
+	if len(key) < minKeyLength {
+		return ErrInvalidKey
+	}
+	c.key = key
+	c.bufSize = bufAdd + len(c.key)
+	if c.buf.Cap() < c.bufSize {
+		c.buf.Grow(c.bufSize - c.buf.Cap())
+	}
+	c.scan = 0
+	return nil
+}
+
+// Reset puts the chunkio stream back into a readable state.  This can be used
+// when the end of a chunk is reached to enable reading the next chunk.
+func (c *Reader) Reset() {
+	if c.buf.Len() == 0 && c.ierr != nil {
+		c.err = io.ErrUnexpectedEOF
+	} else {
+		c.err = nil
+	}
+	c.scan = 0
+	c.found = false
+}
+
+func (c *Reader) readScanned(p []byte) (int, error) {
+	var n int
+
+	if c.scan > len(p) {
+		n, _ = c.buf.Read(p)
+	} else {
+		n, _ = c.buf.Read(p[:c.scan])
+	}
+	c.scan = c.scan - n
+	if n > 0 && c.scan >= 0 {
+		return n, nil
+	} else {
+		return 0, io.ErrUnexpectedEOF
+	}
+}
+
+func (c *Reader) readEOF() (int, error) {
+	// Discard key from input stream
+	r := make([]byte, len(c.key))
+	n, err := c.buf.Read(r)
+	if n != len(c.key) || err != nil {
+		panic("Error: Unexpected error in chunkio.readEOF()")
+	}
+	// Set / return EOF
+	c.err = io.EOF
+	return 0, io.EOF
+}
+
+func (c *Reader) bufFill() error {
+	for c.buf.Len() < c.bufSize {
+		t := make([]byte, c.bufSize-c.buf.Len())
+		n, err := c.rd.Read(t)
+		c.buf.Write(t[:n])
+		if err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
+// Read implements the standard Reader interface allowing chunkio to be used
+// anywhere a standard Reader can be used.  Read puts data into p.  It returns
+// the number of bytes read into p.  The bytes are taken from at most one read
+// on the underlying Reader, hence n may be less than len(p).  When the key is
+// reached (EOF for the stream chunk), the count will be zero and err will be
+// io.EOF.  If the key has been set to nil, the Read function performs exactly
+// like the underlying stream Read function (no key scanning).
+func (c *Reader) Read(p []byte) (int, error) {
+	if len(p) == 0 {
+		return 0, nil
+	}
+	if c.err != nil {
+		return 0, c.err
+	}
+	if c.key == nil {
+		if c.buf.Len() > 0 {
+			return c.buf.Read(p)
+		}
+		return c.rd.Read(p)
+	}
+	if c.scan > 0 {
+		return c.readScanned(p)
+	}
+	if c.found {
+		return c.readEOF()
+	}
+	c.ierr = c.bufFill()
+	pos := bytes.Index(c.buf.Bytes(), c.key)
+	switch pos {
+	case -1:
+		if c.ierr != nil {
+			// Reached input EOF w/o key
+			c.scan = c.buf.Len()
+			c.err = io.ErrUnexpectedEOF
+			return c.readScanned(p)
+		}
+		c.scan = c.buf.Len() - len(c.key)
+		if c.scan <= 0 {
+			panic("Error: Unexpected error in chunkio.Read()")
+		}
+		return c.readScanned(p)
+	case 0:
+		c.scan = 0
+		c.found = true
+		return c.readEOF()
+	default:
+		c.scan = pos
+		c.found = true
+		return c.readScanned(p)
+	}
+}

chunkio_test.go

@@ -0,0 +1,217 @@
+// Copyright (c) 2019 Jason T. Lenz.  All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be found
+// in the LICENSE file.
+
+package chunkio_test
+
+import (
+	"bytes"
+	"fmt"
+	"io"
+	"io/ioutil"
+	"math/rand"
+	"git.lenzplace.org/lenzj/chunkio"
+	"strings"
+	"testing"
+)
+
+func ExampleUppercase() {
+	example := []byte("the quick {U}brown fox jumps{L} over the lazy dog")
+	cio := chunkio.NewReader(bytes.NewReader(example))
+	cio.SetKey([]byte("{U}"))
+	s1, _ := ioutil.ReadAll(cio)
+	cio.Reset()
+	cio.SetKey([]byte("{L}"))
+	s2, _ := ioutil.ReadAll(cio)
+	cio.Reset()
+	s3, _ := ioutil.ReadAll(cio)
+	fmt.Print(string(s1) + strings.ToUpper(string(s2)) + string(s3))
+	// Output: the quick BROWN FOX JUMPS over the lazy dog
+}
+
+func TestShortNewReader(t *testing.T) {
+	c := chunkio.NewReader(bytes.NewReader([]byte("")))
+	if c.GetKey() != nil {
+		t.Errorf("GetKey. Expected \"%v\", got \"%v\"", nil, c.GetKey())
+	}
+	if c.GetErr() != nil {
+		t.Errorf("GetErr. Expected \"%v\", got \"%v\"", nil, c.GetErr())
+	}
+	resid, err := ioutil.ReadAll(c)
+	if bytes.Compare(resid, []byte("")) != 0 {
+		t.Errorf("Read raw Bufio. Expected \"%v\", got \"%v\"", []byte(""), resid)
+	}
+	if err != nil {
+		t.Errorf("Read raw Bufio. Expected error code \"%v\", got \"%v\"", nil, err)
+	}
+}
+
+func TestShortSetKey(t *testing.T) {
+	c := chunkio.NewReader(bytes.NewReader([]byte("")))
+	c.SetKey([]byte("123"))
+	if bytes.Compare(c.GetKey(), []byte("123")) != 0 {
+		t.Errorf("SetKey. Expected key to be \"%v\", got \"%v\"", []byte("123"), c.GetKey())
+	}
+}
+
+func TestShortRead(t *testing.T) {
+	cases := []struct {
+		desc string
+		in   []byte
+		key1 []byte
+		out1 []byte
+		err1 error
+		key2 []byte
+		out2 []byte
+		err2 error
+	}{
+		{
+			desc: "No key detected",
+			key1: []byte("123456"),
+			in:   []byte("---\nauthor : Jason\n---\nqwerty"),
+			out1: []byte("---\nauthor : Jason\n---\nqwerty"),
+			err1: io.ErrUnexpectedEOF,
+			key2: []byte("123456"),
+			out2: []byte(""),
+			err2: io.ErrUnexpectedEOF,
+		},
+		{
+			desc: "Simple key detected at start",
+			in:   []byte("---\nauthor : Jason\n---\nqwerty"),
+			key1: []byte("---\n"),
+			out1: []byte(""),
+			err1: nil,
+			key2: []byte("---\n"),
+			out2: []byte("author : Jason\n"),
+			err2: nil,
+		},
+		{
+			desc: "Simple key detected mid stream",
+			in:   []byte("ytrewq\n---\nauthor : Jason"),
+			key1: []byte("---\n"),
+			out1: []byte("ytrewq\n"),
+			err1: nil,
+			key2: []byte("---\n"),
+			out2: []byte("author : Jason"),
+			err2: io.ErrUnexpectedEOF,
+		},
+		{
+			desc: "Simple key detected mid stream then set key to nil",
+			in:   []byte("ytrewq\n---\nauthor : Jason"),
+			key1: []byte("---\n"),
+			out1: []byte("ytrewq\n"),
+			err1: nil,
+			key2: nil,
+			out2: []byte("author : Jason"),
+			err2: nil,
+		},
+		{
+			desc: "Empty input stream",
+			in:   []byte(""),
+			key1: []byte("---\n"),
+			out1: []byte(""),
+			err1: io.ErrUnexpectedEOF,
+			key2: nil,
+			out2: []byte(""),
+			err2: io.ErrUnexpectedEOF,
+		},
+	}
+	for _, c := range cases {
+		rd := chunkio.NewReader(bytes.NewReader(c.in))
+		rd.SetKey(c.key1)
+		out1, err1 := ioutil.ReadAll(rd)
+		if c.err1 != err1 {
+			t.Errorf("Case %q. Expected stream read error=\"%v\", got \"%v\"", c.desc, c.err1, err1)
+		}
+		if bytes.Compare(c.out1, out1) != 0 {
+			t.Errorf("Case %q. Expected stream read=%q, got %q", c.desc, c.out1, out1)
+		}
+		rd.Reset()
+		rd.SetKey(c.key2)
+		out2, err2 := ioutil.ReadAll(rd)
+		if c.err2 != err2 {
+			t.Errorf("Case %q. Expected 2nd stream read error=\"%v\", got \"%v\"", c.desc, c.err2, err2)
+		}
+		if bytes.Compare(c.out2, out2) != 0 {
+			t.Errorf("Case %q. Expected 2nd stream read=%q, got %q", c.desc, c.out2, out2)
+		}
+	}
+}
+
+// Test each input length from zero up to a large number.
+func TestLongReadSizes(t *testing.T) {
+	for i := 0; i < 20000; i++ {
+		rd := chunkio.NewReader(bytes.NewReader(append(bytes.Repeat([]byte("X"), i), []byte(";;;")...)))
+		rd.SetKey([]byte(";;;"))
+		out, err := ioutil.ReadAll(rd)
+		if len(out) != i {
+			t.Errorf("Failed.  Read byte sequence size %d instead of %d", len(out), i)
+		}
+		if err != nil {
+			t.Errorf("Failed.  Attempted to read byte sequence size %q and got error %v", i, err)
+		}
+	}
+}
+
+// Test a large number of cases randomizing the following:
+// * key (length and contents)
+// * input data before key (length and contents)
+// * additional "garbage" data after key (length and contents)
+func TestLongReadRand(t *testing.T) {
+	const (
+		numcycles = 8000
+		maxinput  = 4096 * 10
+		maxread   = maxinput / 10
+		maxkey    = maxinput / 12
+	)
+
+	var (
+		key     []byte // Key sequence to use
+		input   []byte // Source sequence
+		output  []byte // Destination sequence
+		garbage []byte // Garbage sequence at end
+		readbuf []byte // Buffer to store each Read call in
+		num     int    // Number of bytes from Read
+		err     error  // Error return code from Read
+		cio     *chunkio.Reader
+	)
+
+	for i := 0; i < numcycles; i++ {
+		output = nil
+		key = make([]byte, rand.Intn(maxkey-1)+1)
+		rand.Read(key)
+
+		input = make([]byte, rand.Intn(maxinput-1)+1)
+		rand.Read(input)
+
+		// Make sure key doesn't exist in input stream
+		for {
+			p := bytes.Index(input, key)
+			if p == -1 {
+				break
+			}
+			input[p] = input[p] + 1
+		}
+
+		garbage = make([]byte, rand.Intn(maxinput-1)+1)
+		rand.Read(garbage)
+
+		cio = chunkio.NewReader(bytes.NewReader(append(append(input, key...), garbage...)))
+		cio.SetKey(key)
+		for {
+			readbuf = make([]byte, rand.Intn(maxread-1)+1)
+			num, err = cio.Read(readbuf)
+			if err == nil {
+				output = append(output, readbuf[:num]...)
+			} else {
+				break
+			}
+		}
+		if bytes.Compare(input, output) != 0 {
+			t.Errorf("Cycle %d failed. Input and output differ!", i)
+		}
+		if len(output) != len(input) {
+			t.Errorf("Cycle %d failed. Read size should be %d but was %d", i, len(input), len(output))
+		}
+	}
+}

go.mod

@@ -0,0 +1,3 @@
+module git.lenzplace.org/lenzj/chunkio
+
+go 1.12

template/Makefile.got

@@ -0,0 +1,39 @@
+;;;
+{
+    "pgotInclude" : [
+        "global.got",
+        "Makefile.src/mk-goLib.got",
+        "Makefile.src/mk-docMain.got",
+        "Makefile.src/mk-mkFile.got",
+        "Makefile.src/mk-lint.got",
+        "Makefile.src/mk-todo.got"
+    ]
+}
+;;;
+.POSIX:
+
+PNAME = chunkio
+
+RTEMPLATE ?= ../repo-template
+
+all: doc mkFile
+
+doc: docMain
+
+cleanDoc: cleanDocMain
+
+.DEFAULT_GOAL := all
+
+.PHONY: all doc cleanDoc
+
+{{template "mk-goLib" .}}
+
+{{template "mk-docMain" .}}
+
+{{template "mk-mkFile" .}}
+
+{{template "mk-lint" .}}
+
+{{template "mk-todo" .}}
+
+# vim:set noet tw=80:

template/README.md.got

@@ -0,0 +1,95 @@
+;;;
+{
+  "rname": "chunkio",
+  "pgotInclude": [ "README.src/all.got" ]
+}
+;;;
+# {{.rname}}
+
+**{{.rname}}** is a golang package that provides functionality for transparently
+reading a subset of a stream containing a user defined ending byte sequence.
+When the byte sequence is reached an EOF is returned. This sub stream can be
+accessed or passed to other routines as standard Reader objects.
+
+## Interface
+
+###Variables
+
+```text
+var ErrInvalidKey = errors.New("{{.rname}}: invalid key definition")
+```
+
+###Types
+
+```text
+type Reader struct {
+    // Has unexported fields.
+}
+    Reader implements {{.rname}} functionality wrapped around an io.Reader object
+
+func NewReader(rd io.Reader) *Reader
+    NewReader creates a new chunk reader
+
+func (c *Reader) GetErr() error
+    GetErr returns the error status for the current active {{.rname}} stream
+
+func (c *Reader) GetKey() []byte
+    GetKey returns the key for the current active {{.rname}} stream
+
+func (c *Reader) Read(p []byte) (int, error)
+    Read implements the standard Reader interface allowing {{.rname}} to be used
+    anywhere a standard Reader can be used. Read reads data into p. It returns
+    the number of bytes read into p. The bytes are taken from at most one Read
+    on the underlying Reader, hence n may be less than len(p). When the key is
+    reached (EOF for the stream chunk), the count will be zero and err will be
+    io.EOF. If the key has been set to nil, the Read function performs exactly
+    like the underlying stream Read function (no key scanning).
+
+func (c *Reader) Reset()
+    Reset puts the {{.rname}} stream back into a readable state. This can be used
+    when the end of a chunk is reached to enable reading the next chunk.
+
+func (c *Reader) SetKey(key []byte) error
+    SetKey updates the search key. The search key can also be cleared by
+    providing a nil key.
+```
+
+## Example usage.
+
+```go
+import (
+        "bytes"
+        "fmt"
+        "io/ioutil"
+        "git.lenzplace.org/lenzj/{{.rname}}"
+        "strings"
+)
+
+func ExampleUppercase() {
+        example := []byte("the quick {U}brown fox jumps{R} over the lazy dog")
+        cio := {{.rname}}.NewReader(bytes.NewReader(example))
+        cio.SetKey([]byte("{U}"))
+        s1, _ := ioutil.ReadAll(cio)
+        cio.Reset()
+        cio.SetKey([]byte("{R}"))
+        s2, _ := ioutil.ReadAll(cio)
+        cio.Reset()
+        s3, _ := ioutil.ReadAll(cio)
+        fmt.Print(string(s1)+strings.ToUpper(string(s2))+string(s3))
+        // Output: the quick BROWN FOX JUMPS over the lazy dog
+}
+```
+
+## Running the tests
+
+```
+$ make check
+```
+
+{{template "rd-contributing" .}}
+
+{{template "rd-versioning" .}}
+
+{{template "rd-license" .}}
+
+<!-- vim:set ts=4 sw=4 et tw=80: -->