Adds bzip2 support to post-processor (#10867)

* compress post processor: add bzip2 + tests * post-processor/compress/post-processor_test.go: refactor tests and add tests for bzip2 * post-processor_test.go: test write/read for all compression algos * check artifact.Destroy() errors * close archive before deleting it Co-authored-by: Adrien Delorme <[email protected]>
2026-10-11 08:51:40 -04:00 · 2021-04-15 18:05:09 +02:00
parent c17f236e85
commit a6c5958c67
42 changed files with 5136 additions and 82 deletions
@@ -25,6 +25,7 @@ require (
 	github.com/dgrijalva/jwt-go v3.2.0+incompatible
 	github.com/digitalocean/go-qemu v0.0.0-20201211181942-d361e7b4965f
 	github.com/digitalocean/godo v1.11.1
+	github.com/dsnet/compress v0.0.1
 	github.com/exoscale/packer-plugin-exoscale v0.1.1
 	github.com/fatih/camelcase v1.0.0
 	github.com/fatih/structtag v1.0.0
@@ -81,7 +82,7 @@ require (
 	github.com/tencentcloud/tencentcloud-sdk-go v3.0.222+incompatible
 	github.com/ucloud/ucloud-sdk-go v0.16.3
 	github.com/ufilesdk-dev/ufile-gosdk v0.0.0-20190830075812-b4dbc4ef43a6
-	github.com/ulikunitz/xz v0.5.5
+	github.com/ulikunitz/xz v0.5.6
 	github.com/vmware/govmomi v0.23.1
 	github.com/xanzy/go-cloudstack v0.0.0-20190526095453-42f262b63ed0
 	github.com/yandex-cloud/go-genproto v0.0.0-20200915125933-33de72a328bd
@@ -200,6 +200,9 @@ github.com/dimchansky/utfbom v1.1.0 h1:FcM3g+nofKgUteL8dm/UpdRXNC9KmADgTpLKsu0TR
 github.com/dimchansky/utfbom v1.1.0/go.mod h1:rO41eb7gLfo8SF1jd9F8HplJm1Fewwi4mQvIirEdv+8=
 github.com/dnaeon/go-vcr v1.0.1 h1:r8L/HqC0Hje5AXMu1ooW8oyQyOFv4GxqpL0nRP7SLLY=
 github.com/dnaeon/go-vcr v1.0.1/go.mod h1:aBB1+wY4s93YsC3HHjMBMrwTj2R9FHDzUr9KyGc8n1E=
+github.com/dsnet/compress v0.0.1 h1:PlZu0n3Tuv04TzpfPbrnI0HW/YwodEXDS+oPKahKF0Q=
+github.com/dsnet/compress v0.0.1/go.mod h1:Aw8dCMJ7RioblQeTqt88akK31OvO8Dhf5JflhBbQEHo=
+github.com/dsnet/golib v0.0.0-20171103203638-1ea166775780/go.mod h1:Lj+Z9rebOhdfkVLjJ8T6VcRQv3SXugXy999NBtR9aFY=
 github.com/dylanmei/iso8601 v0.1.0 h1:812NGQDBcqquTfH5Yeo7lwR0nzx/cKdsmf3qMjPURUI=
 github.com/dylanmei/iso8601 v0.1.0/go.mod h1:w9KhXSgIyROl1DefbMYIE7UVSIvELTbMrCfx+QkYnoQ=
 github.com/dylanmei/winrmtest v0.0.0-20170819153634-c2fbb09e6c08 h1:0bp6/GrNOrTDtSXe9YYGCwf8jp5Fb/b+4a6MTRm4qzY=
@@ -445,12 +448,6 @@ github.com/hashicorp/packer-plugin-sdk v0.0.14/go.mod h1:tNb3XzJPnjMl3QuUdKmF47B
 github.com/hashicorp/packer-plugin-sdk v0.1.0/go.mod h1:CFsC20uZjtER/EnTn/CSMKD0kEdkqOVev8mtOmfnZiI=
 github.com/hashicorp/packer-plugin-sdk v0.1.1/go.mod h1:1d3nqB9LUsXMQaNUiL67Q+WYEtjsVcLNTX8ikVlpBrc=
 github.com/hashicorp/packer-plugin-sdk v0.1.2/go.mod h1:KRjczE1/c9NV5Re+PXt3myJsVTI/FxEHpZjRjOH0Fug=
-github.com/hashicorp/packer-plugin-sdk v0.1.3-0.20210407090040-d1eff9fe99e8 h1:pkB+Y15/ck/NRUBFF9DrdPYQwmnHsEvnNwmgMfl/8hA=
-github.com/hashicorp/packer-plugin-sdk v0.1.3-0.20210407090040-d1eff9fe99e8/go.mod h1:xePpgQgQYv/bamiypx3hH9ukidxDdcN8q0R0wLi8IEQ=
-github.com/hashicorp/packer-plugin-sdk v0.1.3-0.20210407130359-85b84b1d6060 h1:uRrDQYiP3pFn5W17Bvj9If2taHB/DqIP7uuPQGnLDFM=
-github.com/hashicorp/packer-plugin-sdk v0.1.3-0.20210407130359-85b84b1d6060/go.mod h1:xePpgQgQYv/bamiypx3hH9ukidxDdcN8q0R0wLi8IEQ=
-github.com/hashicorp/packer-plugin-sdk v0.1.3-0.20210407130906-826d4f395a10 h1:VlcHJEpR99eeZi7uujdQKFOIK8rE5ditXGqpBWiGjc4=
-github.com/hashicorp/packer-plugin-sdk v0.1.3-0.20210407130906-826d4f395a10/go.mod h1:xePpgQgQYv/bamiypx3hH9ukidxDdcN8q0R0wLi8IEQ=
 github.com/hashicorp/packer-plugin-sdk v0.1.3-0.20210407132324-af39c7839daf h1:0DBlIExTDefzbfkOl213QtgJsVJXWdgW/aIQIvYUMzs=
 github.com/hashicorp/packer-plugin-sdk v0.1.3-0.20210407132324-af39c7839daf/go.mod h1:xePpgQgQYv/bamiypx3hH9ukidxDdcN8q0R0wLi8IEQ=
 github.com/hashicorp/serf v0.8.2/go.mod h1:6hOLApaqBFA1NXqRQAsxw9QxuDEvNxSQRwA/JwenrHc=
@@ -496,10 +493,12 @@ github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfV
 github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvWXihfKN4Q=
 github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck=
 github.com/klauspost/compress v0.0.0-20160131094358-f86d2e6d8a77/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A=
+github.com/klauspost/compress v1.4.1/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A=
 github.com/klauspost/compress v1.11.6/go.mod h1:aoV0uJVorq1K+umq18yTdKaF57EivdYsUV+/s2qKfXs=
 github.com/klauspost/compress v1.11.7 h1:0hzRabrMN4tSTvMfnL3SCv1ZGeAP23ynzodBgaHeMeg=
 github.com/klauspost/compress v1.11.7/go.mod h1:aoV0uJVorq1K+umq18yTdKaF57EivdYsUV+/s2qKfXs=
 github.com/klauspost/cpuid v0.0.0-20160106104451-349c67577817/go.mod h1:Pj4uuM528wm8OyEC2QMXAi2YiTZ96dNQPGgoMS4s3ek=
+github.com/klauspost/cpuid v1.2.0/go.mod h1:Pj4uuM528wm8OyEC2QMXAi2YiTZ96dNQPGgoMS4s3ek=
 github.com/klauspost/crc32 v0.0.0-20160114101742-999f3125931f/go.mod h1:+ZoRqAPRLkC4NPOvfYeR5KNOrY6TD+/sAC3HXPZgDYg=
 github.com/klauspost/crc32 v1.2.0 h1:0VuyqOCruD33/lJ/ojXNvzVyl8Zr5zdTmj9l9qLZ86I=
 github.com/klauspost/crc32 v1.2.0/go.mod h1:+ZoRqAPRLkC4NPOvfYeR5KNOrY6TD+/sAC3HXPZgDYg=
@@ -686,8 +685,9 @@ github.com/ugorji/go v1.2.4 h1:cTciPbZ/VSOzCLKclmssnfQ/jyoVyOcJ3aoJyUV1Urc=
 github.com/ugorji/go v1.2.4/go.mod h1:EuaSCk8iZMdIspsu6HXH7X2UGKw1ezO4wCfGszGmmo4=
 github.com/ugorji/go/codec v1.2.4 h1:C5VurWRRCKjuENsbM6GYVw8W++WVW9rSxoACKIvxzz8=
 github.com/ugorji/go/codec v1.2.4/go.mod h1:bWBu1+kIRWcF8uMklKaJrR6fTWQOwAlrIzX22pHwryA=
-github.com/ulikunitz/xz v0.5.5 h1:pFrO0lVpTBXLpYw+pnLj6TbvHuyjXMfjGeCwSqCVwok=
 github.com/ulikunitz/xz v0.5.5/go.mod h1:2bypXElzHzzJZwzH67Y6wb67pO62Rzfn7BSiF4ABRW8=
+github.com/ulikunitz/xz v0.5.6 h1:jGHAfXawEGZQ3blwU5wnWKQJvAraT7Ftq9EXjnXYgt8=
+github.com/ulikunitz/xz v0.5.6/go.mod h1:2bypXElzHzzJZwzH67Y6wb67pO62Rzfn7BSiF4ABRW8=
 github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc=
 github.com/valyala/fasttemplate v1.0.1/go.mod h1:UQGH1tvbgY+Nz5t2n7tXsz52dQxojPUpymEIMZ47gx8=
 github.com/valyala/fasttemplate v1.1.0/go.mod h1:UQGH1tvbgY+Nz5t2n7tXsz52dQxojPUpymEIMZ47gx8=
@@ -14,6 +14,7 @@ import (
 	"runtime"

 	"github.com/biogo/hts/bgzf"
+	"github.com/dsnet/compress/bzip2"
 	"github.com/hashicorp/hcl/v2/hcldec"
 	"github.com/hashicorp/packer-plugin-sdk/common"
 	packersdk "github.com/hashicorp/packer-plugin-sdk/packer"
@@ -160,6 +161,14 @@ func (p *PostProcessor) PostProcess(
 			return nil, false, false, fmt.Errorf(errTmpl, p.config.Algorithm, err)
 		}
 		defer output.Close()
+	case "bzip2":
+		ui.Say(fmt.Sprintf("Using bzip2 compression with 1 core for %s (library does not support MT)",
+			target))
+		output, err = makeBZIP2Writer(outputFile, p.config.CompressionLevel)
+		if err != nil {
+			return nil, false, false, fmt.Errorf(errTmpl, p.config.Algorithm, err)
+		}
+		defer output.Close()
 	case "lz4":
 		ui.Say(fmt.Sprintf("Using lz4 compression with %d cores for %s",
 			runtime.GOMAXPROCS(-1), target))
@@ -242,12 +251,13 @@ func (config *Config) detectFromFilename() {
 	var result [][]string

 	extensions := map[string]string{
-		"tar":  "tar",
-		"zip":  "zip",
-		"gz":   "pgzip",
-		"lz4":  "lz4",
-		"bgzf": "bgzf",
-		"xz":   "xz",
+		"tar":   "tar",
+		"zip":   "zip",
+		"gz":    "pgzip",
+		"lz4":   "lz4",
+		"bgzf":  "bgzf",
+		"xz":    "xz",
+		"bzip2": "bzip2",
 	}

 	if config.Format == "" {
@@ -304,6 +314,20 @@ func makeBGZFWriter(output io.WriteCloser, compressionLevel int) (io.WriteCloser
 	return bgzfWriter, nil
 }

+func makeBZIP2Writer(output io.Writer, compressionLevel int) (io.WriteCloser, error) {
+	// Set the default to highest level compression
+	bzipCFG := &bzip2.WriterConfig{Level: 9}
+	// Override our set defaults
+	if compressionLevel > 0 {
+		bzipCFG.Level = compressionLevel
+	}
+	bzipWriter, err := bzip2.NewWriter(output, bzipCFG)
+	if err != nil {
+		return nil, err
+	}
+	return bzipWriter, nil
+}
+
 func makeLZ4Writer(output io.WriteCloser, compressionLevel int) (io.WriteCloser, error) {
 	lzwriter := lz4.NewWriter(output)
 	if compressionLevel > 0 {
@@ -1,6 +1,8 @@
 package compress

 import (
+	"archive/tar"
+	"archive/zip"
 	"compress/gzip"
 	"context"
 	"fmt"
@@ -9,9 +11,11 @@ import (
 	"strings"
 	"testing"

+	"github.com/dsnet/compress/bzip2"
 	packersdk "github.com/hashicorp/packer-plugin-sdk/packer"
 	"github.com/hashicorp/packer-plugin-sdk/template"
 	"github.com/hashicorp/packer/builder/file"
+	"github.com/pierrec/lz4"
 )

 func TestDetectFilename(t *testing.T) {
@@ -58,73 +62,6 @@ func TestDetectFilename(t *testing.T) {

 const expectedFileContents = "Hello world!"

-func TestSimpleCompress(t *testing.T) {
-	const config = `
-	{
-	    "post-processors": [
-	        {
-	            "type": "compress",
-	            "output": "package.tar.gz"
-	        }
-	    ]
-	}
-	`
-	artifact := testArchive(t, config)
-	defer artifact.Destroy()
-
-	fi, err := os.Stat("package.tar.gz")
-	if err != nil {
-		t.Errorf("Unable to read archive: %s", err)
-	}
-	if fi.IsDir() {
-		t.Error("Archive should not be a directory")
-	}
-}
-
-func TestZipArchive(t *testing.T) {
-	const config = `
-	{
-	    "post-processors": [
-	        {
-	            "type": "compress",
-	            "output": "package.zip"
-	        }
-	    ]
-	}
-	`
-
-	artifact := testArchive(t, config)
-	defer artifact.Destroy()
-
-	// Verify things look good
-	_, err := os.Stat("package.zip")
-	if err != nil {
-		t.Errorf("Unable to read archive: %s", err)
-	}
-}
-
-func TestTarArchive(t *testing.T) {
-	const config = `
-	{
-	    "post-processors": [
-	        {
-	            "type": "compress",
-	            "output": "package.tar"
-	        }
-	    ]
-	}
-	`
-
-	artifact := testArchive(t, config)
-	defer artifact.Destroy()
-
-	// Verify things look good
-	_, err := os.Stat("package.tar")
-	if err != nil {
-		t.Errorf("Unable to read archive: %s", err)
-	}
-}
-
 func TestCompressOptions(t *testing.T) {
 	const config = `
 	{
@@ -246,3 +183,105 @@ func testArchive(t *testing.T, config string) packersdk.Artifact {

 	return artifactOut
 }
+
+func TestArchive(t *testing.T) {
+	tc := map[string]func(*os.File) ([]byte, error){
+		"bzip2": func(archive *os.File) ([]byte, error) {
+			bzipReader, err := bzip2.NewReader(archive, nil)
+			if err != nil {
+				return nil, err
+			}
+			return ioutil.ReadAll(bzipReader)
+		},
+		"zip": func(archive *os.File) ([]byte, error) {
+			fi, _ := archive.Stat()
+			zipReader, err := zip.NewReader(archive, fi.Size())
+			if err != nil {
+				return nil, err
+			}
+			ctt, err := zipReader.File[0].Open()
+			if err != nil {
+				return nil, err
+			}
+			return ioutil.ReadAll(ctt)
+		},
+		"tar": func(archive *os.File) ([]byte, error) {
+			tarReader := tar.NewReader(archive)
+			_, err := tarReader.Next()
+			if err != nil {
+				return nil, err
+			}
+			return ioutil.ReadAll(tarReader)
+		},
+		"tar.gz": func(archive *os.File) ([]byte, error) {
+			gzipReader, err := gzip.NewReader(archive)
+			if err != nil {
+				return nil, err
+			}
+			tarReader := tar.NewReader(gzipReader)
+			_, err = tarReader.Next()
+			if err != nil {
+				return nil, err
+			}
+			return ioutil.ReadAll(tarReader)
+		},
+		"gz": func(archive *os.File) ([]byte, error) {
+			gzipReader, _ := gzip.NewReader(archive)
+			return ioutil.ReadAll(gzipReader)
+		},
+		"lz4": func(archive *os.File) ([]byte, error) {
+			lz4Reader := lz4.NewReader(archive)
+			return ioutil.ReadAll(lz4Reader)
+		},
+	}
+
+	for format, unzip := range tc {
+		t.Run(format, func(t *testing.T) {
+			config := fmt.Sprintf(`
+		{
+			"post-processors": [
+				{
+					"type": "compress",
+					"output": "package.%s"
+				}
+			]
+		}
+		`, format)
+
+			artifact := testArchive(t, config)
+			defer func() {
+				err := artifact.Destroy()
+				if err != nil {
+					t.Fatal(err)
+				}
+			}()
+
+			filename := fmt.Sprintf("package.%s", format)
+			// Verify things look good
+			_, err := os.Stat(filename)
+			if err != nil {
+				t.Errorf("Unable to read archive: %s", err)
+			}
+
+			archive, err := os.Open(filename)
+			if err != nil {
+				t.Fatal(err)
+			}
+			defer func() {
+				err := archive.Close()
+				if err != nil {
+					t.Fatal(err)
+				}
+			}()
+
+			found, err := unzip(archive)
+			if err != nil {
+				t.Fatal(err)
+			}
+			if string(found) != expectedFileContents {
+				t.Errorf("Expected:\n%s\nFound:\n%s\n", expectedFileContents, string(found))
+			}
+		})
+	}
+
+}
@@ -0,0 +1,36 @@
+sudo: false
+language: go
+before_install:
+  - curl -L https://github.com/google/brotli/archive/v1.0.2.tar.gz | tar -zxv
+  - (cd brotli-1.0.2 && mkdir out && cd out && ../configure-cmake && make && sudo make install)
+  - rm -rf brotli-1.0.2
+  - curl -L https://github.com/facebook/zstd/archive/v1.3.2.tar.gz | tar -zxv
+  - (cd zstd-1.3.2 && sudo make install)
+  - rm -rf zstd-1.3.2
+  - sudo ldconfig
+  - mkdir /tmp/go1.12
+  - curl -L -s https://dl.google.com/go/go1.12.linux-amd64.tar.gz | tar -zxf - -C /tmp/go1.12 --strip-components 1
+  - unset GOROOT
+  - (GO111MODULE=on /tmp/go1.12/bin/go mod vendor)
+  - (cd /tmp && GO111MODULE=on /tmp/go1.12/bin/go get golang.org/x/lint/golint@8f45f776aaf18cebc8d65861cc70c33c60471952)
+  - (cd /tmp && GO111MODULE=on /tmp/go1.12/bin/go get honnef.co/go/tools/cmd/[email protected])
+matrix:
+  include:
+    - go: 1.9.x
+      script:
+        - go test -v -race ./...
+    - go: 1.10.x
+      script:
+        - go test -v -race ./...
+    - go: 1.11.x
+      script:
+        - go test -v -race ./...
+    - go: 1.12.x
+      script:
+        - ./ztest.sh
+    - go: master
+      script:
+        - go test -v -race ./...
+  allow_failures:
+    - go: master
+  fast_finish: true
@@ -0,0 +1,24 @@
+Copyright © 2015, Joe Tsai and The Go Authors. All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are met:
+
+* Redistributions of source code must retain the above copyright notice, this
+list of conditions and the following disclaimer.
+* Redistributions in binary form must reproduce the above copyright notice,
+this list of conditions and the following disclaimer in the documentation and/or
+other materials provided with the distribution.
+* Neither the copyright holder nor the names of its contributors may be used to
+endorse or promote products derived from this software without specific prior
+written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
+ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER BE LIABLE FOR ANY
+DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
+ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
@@ -0,0 +1,75 @@
+# Collection of compression libraries for Go #
+
+[![GoDoc](https://godoc.org/github.com/dsnet/compress/cmp?status.svg)](https://godoc.org/github.com/dsnet/compress)
+[![Build Status](https://travis-ci.org/dsnet/compress.svg?branch=master)](https://travis-ci.org/dsnet/compress)
+[![Report Card](https://goreportcard.com/badge/github.com/dsnet/compress)](https://goreportcard.com/report/github.com/dsnet/compress)
+
+## Introduction ##
+
+**NOTE: This library is in active development. As such, there are no guarantees about the stability of the API. The author reserves the right to arbitrarily break the API for any reason.**
+
+This repository hosts a collection of compression related libraries. The goal of this project is to provide pure Go implementations for popular compression algorithms beyond what the Go standard library provides. The goals for these packages are as follows:
+* Maintainable: That the code remains well documented, well tested, readable, easy to maintain, and easy to verify that it conforms to the specification for the format being implemented.
+* Performant: To be able to compress and decompress within at least 80% of the rates that the C implementations are able to achieve.
+* Flexible: That the code provides low-level and fine granularity control over the compression streams similar to what the C APIs would provide.
+
+Of these three, the first objective is often at odds with the other two objectives and provides interesting challenges. Higher performance can often be achieved by muddling abstraction layers or using non-intuitive low-level primitives. Also, more features and functionality, while useful in some situations, often complicates the API. Thus, this package will attempt to satisfy all the goals, but will defer to favoring maintainability when the performance or flexibility benefits are not significant enough.
+
+
+## Library Status ##
+
+For the packages available, only some features are currently implemented:
+
+| Package | Reader | Writer |
+| ------- | :----: | :----: |
+| brotli | :white_check_mark: | |
+| bzip2 | :white_check_mark: | :white_check_mark: |
+| flate | :white_check_mark: | |
+| xflate | :white_check_mark: | :white_check_mark: |
+
+This library is in active development. As such, there are no guarantees about the stability of the API. The author reserves the right to arbitrarily break the API for any reason. When the library becomes more mature, it is planned to eventually conform to some strict versioning scheme like [Semantic Versioning](http://semver.org/).
+
+However, in the meanwhile, this library does provide some basic API guarantees. For the types defined below, the method signatures are guaranteed to not change. Note that the author still reserves the right to change the fields within each ```Reader``` and ```Writer``` structs.
+```go
+type ReaderConfig struct { ... }
+type Reader struct { ... }
+  func NewReader(io.Reader, *ReaderConfig) (*Reader, error) { ... }
+  func (*Reader) Read([]byte) (int, error)                  { ... }
+  func (*Reader) Close() error                              { ... }
+
+type WriterConfig struct { ... }
+type Writer struct { ... }
+  func NewWriter(io.Writer, *WriterConfig) (*Writer, error) { ... }
+  func (*Writer) Write([]byte) (int, error)                 { ... }
+  func (*Writer) Close() error                              { ... }
+```
+
+To see what work still remains, see the [Task List](https://github.com/dsnet/compress/wiki/Task-List).
+
+## Performance  ##
+
+See [Performance Metrics](https://github.com/dsnet/compress/wiki/Performance-Metrics).
+
+
+## Frequently Asked Questions ##
+
+See [Frequently Asked Questions](https://github.com/dsnet/compress/wiki/Frequently-Asked-Questions).
+
+
+## Installation ##
+
+Run the command:
+
+```go get -u github.com/dsnet/compress```
+
+This library requires `Go1.9` or higher in order to build.
+
+
+## Packages ##
+
+| Package | Description |
+| :------ | :---------- |
+| [brotli](http://godoc.org/github.com/dsnet/compress/brotli) | Package brotli implements the Brotli format, described in RFC 7932. |
+| [bzip2](http://godoc.org/github.com/dsnet/compress/bzip2) | Package bzip2 implements the BZip2 compressed data format. |
+| [flate](http://godoc.org/github.com/dsnet/compress/flate) | Package flate implements the DEFLATE format, described in RFC 1951. |
+| [xflate](http://godoc.org/github.com/dsnet/compress/xflate) | Package xflate implements the XFLATE format, an random-access extension to DEFLATE. |
@@ -0,0 +1,74 @@
+// Copyright 2015, Joe Tsai. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE.md file.
+
+// Package compress is a collection of compression libraries.
+package compress
+
+import (
+	"bufio"
+	"io"
+
+	"github.com/dsnet/compress/internal/errors"
+)
+
+// The Error interface identifies all compression related errors.
+type Error interface {
+	error
+	CompressError()
+
+	// IsDeprecated reports the use of a deprecated and unsupported feature.
+	IsDeprecated() bool
+
+	// IsCorrupted reports whether the input stream was corrupted.
+	IsCorrupted() bool
+}
+
+var _ Error = errors.Error{}
+
+// ByteReader is an interface accepted by all decompression Readers.
+// It guarantees that the decompressor never reads more data than is necessary
+// from the underlying io.Reader.
+type ByteReader interface {
+	io.Reader
+	io.ByteReader
+}
+
+var _ ByteReader = (*bufio.Reader)(nil)
+
+// BufferedReader is an interface accepted by all decompression Readers.
+// It guarantees that the decompressor never reads more data than is necessary
+// from the underlying io.Reader. Since BufferedReader allows a decompressor
+// to peek at bytes further along in the stream without advancing the read
+// pointer, decompression can experience a significant performance gain when
+// provided a reader that satisfies this interface. Thus, a decompressor will
+// prefer this interface over ByteReader for performance reasons.
+//
+// The bufio.Reader satisfies this interface.
+type BufferedReader interface {
+	io.Reader
+
+	// Buffered returns the number of bytes currently buffered.
+	//
+	// This value becomes invalid following the next Read/Discard operation.
+	Buffered() int
+
+	// Peek returns the next n bytes without advancing the reader.
+	//
+	// If Peek returns fewer than n bytes, it also returns an error explaining
+	// why the peek is short. Peek must support peeking of at least 8 bytes.
+	// If 0 <= n <= Buffered(), Peek is guaranteed to succeed without reading
+	// from the underlying io.Reader.
+	//
+	// This result becomes invalid following the next Read/Discard operation.
+	Peek(n int) ([]byte, error)
+
+	// Discard skips the next n bytes, returning the number of bytes discarded.
+	//
+	// If Discard skips fewer than n bytes, it also returns an error.
+	// If 0 <= n <= Buffered(), Discard is guaranteed to succeed without reading
+	// from the underlying io.Reader.
+	Discard(n int) (int, error)
+}
+
+var _ BufferedReader = (*bufio.Reader)(nil)
@@ -0,0 +1,110 @@
+// Copyright 2015, Joe Tsai. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE.md file.
+
+package bzip2
+
+import "github.com/dsnet/compress/bzip2/internal/sais"
+
+// The Burrows-Wheeler Transform implementation used here is based on the
+// Suffix Array by Induced Sorting (SA-IS) methodology by Nong, Zhang, and Chan.
+// This implementation uses the sais algorithm originally written by Yuta Mori.
+//
+// The SA-IS algorithm runs in O(n) and outputs a Suffix Array. There is a
+// mathematical relationship between Suffix Arrays and the Burrows-Wheeler
+// Transform, such that a SA can be converted to a BWT in O(n) time.
+//
+// References:
+//	http://www.hpl.hp.com/techreports/Compaq-DEC/SRC-RR-124.pdf
+//	https://github.com/cscott/compressjs/blob/master/lib/BWT.js
+//	https://www.quora.com/How-can-I-optimize-burrows-wheeler-transform-and-inverse-transform-to-work-in-O-n-time-O-n-space
+type burrowsWheelerTransform struct {
+	buf  []byte
+	sa   []int
+	perm []uint32
+}
+
+func (bwt *burrowsWheelerTransform) Encode(buf []byte) (ptr int) {
+	if len(buf) == 0 {
+		return -1
+	}
+
+	// TODO(dsnet): Find a way to avoid the duplicate input string method.
+	// We only need to do this because suffix arrays (by definition) only
+	// operate non-wrapped suffixes of a string. On the other hand,
+	// the BWT specifically used in bzip2 operate on a strings that wrap-around
+	// when being sorted.
+
+	// Step 1: Concatenate the input string to itself so that we can use the
+	// suffix array algorithm for bzip2's variant of BWT.
+	n := len(buf)
+	bwt.buf = append(append(bwt.buf[:0], buf...), buf...)
+	if cap(bwt.sa) < 2*n {
+		bwt.sa = make([]int, 2*n)
+	}
+	t := bwt.buf[:2*n]
+	sa := bwt.sa[:2*n]
+
+	// Step 2: Compute the suffix array (SA). The input string, t, will not be
+	// modified, while the results will be written to the output, sa.
+	sais.ComputeSA(t, sa)
+
+	// Step 3: Convert the SA to a BWT. Since ComputeSA does not mutate the
+	// input, we have two copies of the input; in buf and buf2. Thus, we write
+	// the transformation to buf, while using buf2.
+	var j int
+	buf2 := t[n:]
+	for _, i := range sa {
+		if i < n {
+			if i == 0 {
+				ptr = j
+				i = n
+			}
+			buf[j] = buf2[i-1]
+			j++
+		}
+	}
+	return ptr
+}
+
+func (bwt *burrowsWheelerTransform) Decode(buf []byte, ptr int) {
+	if len(buf) == 0 {
+		return
+	}
+
+	// Step 1: Compute cumm, where cumm[ch] reports the total number of
+	// characters that precede the character ch in the alphabet.
+	var cumm [256]int
+	for _, v := range buf {
+		cumm[v]++
+	}
+	var sum int
+	for i, v := range cumm {
+		cumm[i] = sum
+		sum += v
+	}
+
+	// Step 2: Compute perm, where perm[ptr] contains a pointer to the next
+	// byte in buf and the next pointer in perm itself.
+	if cap(bwt.perm) < len(buf) {
+		bwt.perm = make([]uint32, len(buf))
+	}
+	perm := bwt.perm[:len(buf)]
+	for i, b := range buf {
+		perm[cumm[b]] = uint32(i)
+		cumm[b]++
+	}
+
+	// Step 3: Follow each pointer in perm to the next byte, starting with the
+	// origin pointer.
+	if cap(bwt.buf) < len(buf) {
+		bwt.buf = make([]byte, len(buf))
+	}
+	buf2 := bwt.buf[:len(buf)]
+	i := perm[ptr]
+	for j := range buf2 {
+		buf2[j] = buf[i]
+		i = perm[i]
+	}
+	copy(buf, buf2)
+}
@@ -0,0 +1,110 @@
+// Copyright 2015, Joe Tsai. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE.md file.
+
+// Package bzip2 implements the BZip2 compressed data format.
+//
+// Canonical C implementation:
+//	http://bzip.org
+//
+// Unofficial format specification:
+//	https://github.com/dsnet/compress/blob/master/doc/bzip2-format.pdf
+package bzip2
+
+import (
+	"fmt"
+	"hash/crc32"
+
+	"github.com/dsnet/compress/internal"
+	"github.com/dsnet/compress/internal/errors"
+)
+
+// There does not exist a formal specification of the BZip2 format. As such,
+// much of this work is derived by either reverse engineering the original C
+// source code or using secondary sources.
+//
+// Significant amounts of fuzz testing is done to ensure that outputs from
+// this package is properly decoded by the C library. Furthermore, we test that
+// both this package and the C library agree about what inputs are invalid.
+//
+// Compression stack:
+//	Run-length encoding 1     (RLE1)
+//	Burrows-Wheeler transform (BWT)
+//	Move-to-front transform   (MTF)
+//	Run-length encoding 2     (RLE2)
+//	Prefix encoding           (PE)
+//
+// References:
+//	http://bzip.org/
+//	https://en.wikipedia.org/wiki/Bzip2
+//	https://code.google.com/p/jbzip2/
+
+const (
+	BestSpeed          = 1
+	BestCompression    = 9
+	DefaultCompression = 6
+)
+
+const (
+	hdrMagic = 0x425a         // Hex of "BZ"
+	blkMagic = 0x314159265359 // BCD of PI
+	endMagic = 0x177245385090 // BCD of sqrt(PI)
+
+	blockSize = 100000
+)
+
+func errorf(c int, f string, a ...interface{}) error {
+	return errors.Error{Code: c, Pkg: "bzip2", Msg: fmt.Sprintf(f, a...)}
+}
+
+func panicf(c int, f string, a ...interface{}) {
+	errors.Panic(errorf(c, f, a...))
+}
+
+// errWrap converts a lower-level errors.Error to be one from this package.
+// The replaceCode passed in will be used to replace the code for any errors
+// with the errors.Invalid code.
+//
+// For the Reader, set this to errors.Corrupted.
+// For the Writer, set this to errors.Internal.
+func errWrap(err error, replaceCode int) error {
+	if cerr, ok := err.(errors.Error); ok {
+		if errors.IsInvalid(cerr) {
+			cerr.Code = replaceCode
+		}
+		err = errorf(cerr.Code, "%s", cerr.Msg)
+	}
+	return err
+}
+
+var errClosed = errorf(errors.Closed, "")
+
+// crc computes the CRC-32 used by BZip2.
+//
+// The CRC-32 computation in bzip2 treats bytes as having bits in big-endian
+// order. That is, the MSB is read before the LSB. Thus, we can use the
+// standard library version of CRC-32 IEEE with some minor adjustments.
+//
+// The byte array is used as an intermediate buffer to swap the bits of every
+// byte of the input.
+type crc struct {
+	val uint32
+	buf [256]byte
+}
+
+// update computes the CRC-32 of appending buf to c.
+func (c *crc) update(buf []byte) {
+	cval := internal.ReverseUint32(c.val)
+	for len(buf) > 0 {
+		n := len(buf)
+		if n > len(c.buf) {
+			n = len(c.buf)
+		}
+		for i, b := range buf[:n] {
+			c.buf[i] = internal.ReverseLUT[b]
+		}
+		cval = crc32.Update(cval, crc32.IEEETable, c.buf[:n])
+		buf = buf[n:]
+	}
+	c.val = internal.ReverseUint32(cval)
+}
@@ -0,0 +1,13 @@
+// Copyright 2016, Joe Tsai. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE.md file.
+
+// +build !gofuzz
+
+// This file exists to suppress fuzzing details from release builds.
+
+package bzip2
+
+type fuzzReader struct{}
+
+func (*fuzzReader) updateChecksum(int64, uint32) {}
@@ -0,0 +1,77 @@
+// Copyright 2016, Joe Tsai. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE.md file.
+
+// +build gofuzz
+
+// This file exists to export internal implementation details for fuzz testing.
+
+package bzip2
+
+func ForwardBWT(buf []byte) (ptr int) {
+	var bwt burrowsWheelerTransform
+	return bwt.Encode(buf)
+}
+
+func ReverseBWT(buf []byte, ptr int) {
+	var bwt burrowsWheelerTransform
+	bwt.Decode(buf, ptr)
+}
+
+type fuzzReader struct {
+	Checksums Checksums
+}
+
+// updateChecksum updates Checksums.
+//
+// If a valid pos is provided, it appends the (pos, val) pair to the slice.
+// Otherwise, it will update the last record with the new value.
+func (fr *fuzzReader) updateChecksum(pos int64, val uint32) {
+	if pos >= 0 {
+		fr.Checksums = append(fr.Checksums, Checksum{pos, val})
+	} else {
+		fr.Checksums[len(fr.Checksums)-1].Value = val
+	}
+}
+
+type Checksum struct {
+	Offset int64  // Bit offset of the checksum
+	Value  uint32 // Checksum value
+}
+
+type Checksums []Checksum
+
+// Apply overwrites all checksum fields in d with the ones in cs.
+func (cs Checksums) Apply(d []byte) []byte {
+	d = append([]byte(nil), d...)
+	for _, c := range cs {
+		setU32(d, c.Offset, c.Value)
+	}
+	return d
+}
+
+func setU32(d []byte, pos int64, val uint32) {
+	for i := uint(0); i < 32; i++ {
+		bpos := uint64(pos) + uint64(i)
+		d[bpos/8] &= ^byte(1 << (7 - bpos%8))
+		d[bpos/8] |= byte(val>>(31-i)) << (7 - bpos%8)
+	}
+}
+
+// Verify checks that all checksum fields in d matches those in cs.
+func (cs Checksums) Verify(d []byte) bool {
+	for _, c := range cs {
+		if getU32(d, c.Offset) != c.Value {
+			return false
+		}
+	}
+	return true
+}
+
+func getU32(d []byte, pos int64) (val uint32) {
+	for i := uint(0); i < 32; i++ {
+		bpos := uint64(pos) + uint64(i)
+		val |= (uint32(d[bpos/8] >> (7 - bpos%8))) << (31 - i)
+	}
+	return val
+}
@@ -0,0 +1,28 @@
+// Copyright 2015, Joe Tsai. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE.md file.
+
+// Package sais implements a linear time suffix array algorithm.
+package sais
+
+//go:generate go run sais_gen.go byte sais_byte.go
+//go:generate go run sais_gen.go int sais_int.go
+
+// This package ports the C sais implementation by Yuta Mori. The ports are
+// located in sais_byte.go and sais_int.go, which are identical to each other
+// except for the types. Since Go does not support generics, we use generators to
+// create the two files.
+//
+// References:
+//	https://sites.google.com/site/yuta256/sais
+//	https://www.researchgate.net/publication/221313676_Linear_Time_Suffix_Array_Construction_Using_D-Critical_Substrings
+//	https://www.researchgate.net/publication/224176324_Two_Efficient_Algorithms_for_Linear_Time_Suffix_Array_Construction
+
+// ComputeSA computes the suffix array of t and places the result in sa.
+// Both t and sa must be the same length.
+func ComputeSA(t []byte, sa []int) {
+	if len(sa) != len(t) {
+		panic("mismatching sizes")
+	}
+	computeSA_byte(t, sa, 0, len(t), 256)
+}
@@ -0,0 +1,131 @@
+// Copyright 2015, Joe Tsai. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE.md file.
+
+package bzip2
+
+import "github.com/dsnet/compress/internal/errors"
+
+// moveToFront implements both the MTF and RLE stages of bzip2 at the same time.
+// Any runs of zeros in the encoded output will be replaced by a sequence of
+// RUNA and RUNB symbols are encode the length of the run.
+//
+// The RLE encoding used can actually be encoded to and decoded from using
+// normal two's complement arithmetic. The methodology for doing so is below.
+//
+// Assuming the following:
+//	num: The value being encoded by RLE encoding.
+//	run: A sequence of RUNA and RUNB symbols represented as a binary integer,
+//	where RUNA is the 0 bit, RUNB is the 1 bit, and least-significant RUN
+//	symbols are at the least-significant bit positions.
+//	cnt: The number of RUNA and RUNB symbols.
+//
+// Then the RLE encoding used by bzip2 has this mathematical property:
+//	num+1 == (1<<cnt) | run
+type moveToFront struct {
+	dictBuf [256]uint8
+	dictLen int
+
+	vals    []byte
+	syms    []uint16
+	blkSize int
+}
+
+func (mtf *moveToFront) Init(dict []uint8, blkSize int) {
+	if len(dict) > len(mtf.dictBuf) {
+		panicf(errors.Internal, "alphabet too large")
+	}
+	copy(mtf.dictBuf[:], dict)
+	mtf.dictLen = len(dict)
+	mtf.blkSize = blkSize
+}
+
+func (mtf *moveToFront) Encode(vals []byte) (syms []uint16) {
+	dict := mtf.dictBuf[:mtf.dictLen]
+	syms = mtf.syms[:0]
+
+	if len(vals) > mtf.blkSize {
+		panicf(errors.Internal, "exceeded block size")
+	}
+
+	var lastNum uint32
+	for _, val := range vals {
+		// Normal move-to-front transform.
+		var idx uint8 // Reverse lookup idx in dict
+		for di, dv := range dict {
+			if dv == val {
+				idx = uint8(di)
+				break
+			}
+		}
+		copy(dict[1:], dict[:idx])
+		dict[0] = val
+
+		// Run-length encoding augmentation.
+		if idx == 0 {
+			lastNum++
+			continue
+		}
+		if lastNum > 0 {
+			for rc := lastNum + 1; rc != 1; rc >>= 1 {
+				syms = append(syms, uint16(rc&1))
+			}
+			lastNum = 0
+		}
+		syms = append(syms, uint16(idx)+1)
+	}
+	if lastNum > 0 {
+		for rc := lastNum + 1; rc != 1; rc >>= 1 {
+			syms = append(syms, uint16(rc&1))
+		}
+	}
+	mtf.syms = syms
+	return syms
+}
+
+func (mtf *moveToFront) Decode(syms []uint16) (vals []byte) {
+	dict := mtf.dictBuf[:mtf.dictLen]
+	vals = mtf.vals[:0]
+
+	var lastCnt uint
+	var lastRun uint32
+	for _, sym := range syms {
+		// Run-length encoding augmentation.
+		if sym < 2 {
+			lastRun |= uint32(sym) << lastCnt
+			lastCnt++
+			continue
+		}
+		if lastCnt > 0 {
+			cnt := int((1<<lastCnt)|lastRun) - 1
+			if len(vals)+cnt > mtf.blkSize || lastCnt > 24 {
+				panicf(errors.Corrupted, "run-length decoding exceeded block size")
+			}
+			for i := cnt; i > 0; i-- {
+				vals = append(vals, dict[0])
+			}
+			lastCnt, lastRun = 0, 0
+		}
+
+		// Normal move-to-front transform.
+		val := dict[sym-1] // Forward lookup val in dict
+		copy(dict[1:], dict[:sym-1])
+		dict[0] = val
+
+		if len(vals) >= mtf.blkSize {
+			panicf(errors.Corrupted, "run-length decoding exceeded block size")
+		}
+		vals = append(vals, val)
+	}
+	if lastCnt > 0 {
+		cnt := int((1<<lastCnt)|lastRun) - 1
+		if len(vals)+cnt > mtf.blkSize || lastCnt > 24 {
+			panicf(errors.Corrupted, "run-length decoding exceeded block size")
+		}
+		for i := cnt; i > 0; i-- {
+			vals = append(vals, dict[0])
+		}
+	}
+	mtf.vals = vals
+	return vals
+}
@@ -0,0 +1,274 @@
+// Copyright 2015, Joe Tsai. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE.md file.
+
+package bzip2
+
+import (
+	"io"
+
+	"github.com/dsnet/compress/internal"
+	"github.com/dsnet/compress/internal/errors"
+	"github.com/dsnet/compress/internal/prefix"
+)
+
+type Reader struct {
+	InputOffset  int64 // Total number of bytes read from underlying io.Reader
+	OutputOffset int64 // Total number of bytes emitted from Read
+
+	rd       prefixReader
+	err      error
+	level    int    // The current compression level
+	rdHdrFtr int    // Number of times we read the stream header and footer
+	blkCRC   uint32 // CRC-32 IEEE of each block (as stored)
+	endCRC   uint32 // Checksum of all blocks using bzip2's custom method
+
+	crc crc
+	mtf moveToFront
+	bwt burrowsWheelerTransform
+	rle runLengthEncoding
+
+	// These fields are allocated with Reader and re-used later.
+	treeSels []uint8
+	codes2D  [maxNumTrees][maxNumSyms]prefix.PrefixCode
+	codes1D  [maxNumTrees]prefix.PrefixCodes
+	trees1D  [maxNumTrees]prefix.Decoder
+	syms     []uint16
+
+	fuzzReader // Exported functionality when fuzz testing
+}
+
+type ReaderConfig struct {
+	_ struct{} // Blank field to prevent unkeyed struct literals
+}
+
+func NewReader(r io.Reader, conf *ReaderConfig) (*Reader, error) {
+	zr := new(Reader)
+	zr.Reset(r)
+	return zr, nil
+}
+
+func (zr *Reader) Reset(r io.Reader) error {
+	*zr = Reader{
+		rd: zr.rd,
+
+		mtf: zr.mtf,
+		bwt: zr.bwt,
+		rle: zr.rle,
+
+		treeSels: zr.treeSels,
+		trees1D:  zr.trees1D,
+		syms:     zr.syms,
+	}
+	zr.rd.Init(r)
+	return nil
+}
+
+func (zr *Reader) Read(buf []byte) (int, error) {
+	for {
+		cnt, err := zr.rle.Read(buf)
+		if err != rleDone && zr.err == nil {
+			zr.err = err
+		}
+		if cnt > 0 {
+			zr.crc.update(buf[:cnt])
+			zr.OutputOffset += int64(cnt)
+			return cnt, nil
+		}
+		if zr.err != nil || len(buf) == 0 {
+			return 0, zr.err
+		}
+
+		// Read the next chunk.
+		zr.rd.Offset = zr.InputOffset
+		func() {
+			defer errors.Recover(&zr.err)
+			if zr.rdHdrFtr%2 == 0 {
+				// Check if we are already at EOF.
+				if err := zr.rd.PullBits(1); err != nil {
+					if err == io.ErrUnexpectedEOF && zr.rdHdrFtr > 0 {
+						err = io.EOF // EOF is okay if we read at least one stream
+					}
+					errors.Panic(err)
+				}
+
+				// Read stream header.
+				if zr.rd.ReadBitsBE64(16) != hdrMagic {
+					panicf(errors.Corrupted, "invalid stream magic")
+				}
+				if ver := zr.rd.ReadBitsBE64(8); ver != 'h' {
+					if ver == '0' {
+						panicf(errors.Deprecated, "bzip1 format is not supported")
+					}
+					panicf(errors.Corrupted, "invalid version: %q", ver)
+				}
+				lvl := int(zr.rd.ReadBitsBE64(8)) - '0'
+				if lvl < BestSpeed || lvl > BestCompression {
+					panicf(errors.Corrupted, "invalid block size: %d", lvl*blockSize)
+				}
+				zr.level = lvl
+				zr.rdHdrFtr++
+			} else {
+				// Check and update the CRC.
+				if internal.GoFuzz {
+					zr.updateChecksum(-1, zr.crc.val) // Update with value
+					zr.blkCRC = zr.crc.val            // Suppress CRC failures
+				}
+				if zr.blkCRC != zr.crc.val {
+					panicf(errors.Corrupted, "mismatching block checksum")
+				}
+				zr.endCRC = (zr.endCRC<<1 | zr.endCRC>>31) ^ zr.blkCRC
+			}
+			buf := zr.decodeBlock()
+			zr.rle.Init(buf)
+		}()
+		if zr.InputOffset, err = zr.rd.Flush(); zr.err == nil {
+			zr.err = err
+		}
+		if zr.err != nil {
+			zr.err = errWrap(zr.err, errors.Corrupted)
+			return 0, zr.err
+		}
+	}
+}
+
+func (zr *Reader) Close() error {
+	if zr.err == io.EOF || zr.err == errClosed {
+		zr.rle.Init(nil) // Make sure future reads fail
+		zr.err = errClosed
+		return nil
+	}
+	return zr.err // Return the persistent error
+}
+
+func (zr *Reader) decodeBlock() []byte {
+	if magic := zr.rd.ReadBitsBE64(48); magic != blkMagic {
+		if magic == endMagic {
+			endCRC := uint32(zr.rd.ReadBitsBE64(32))
+			if internal.GoFuzz {
+				zr.updateChecksum(zr.rd.BitsRead()-32, zr.endCRC)
+				endCRC = zr.endCRC // Suppress CRC failures
+			}
+			if zr.endCRC != endCRC {
+				panicf(errors.Corrupted, "mismatching stream checksum")
+			}
+			zr.endCRC = 0
+			zr.rd.ReadPads()
+			zr.rdHdrFtr++
+			return nil
+		}
+		panicf(errors.Corrupted, "invalid block or footer magic")
+	}
+
+	zr.crc.val = 0
+	zr.blkCRC = uint32(zr.rd.ReadBitsBE64(32))
+	if internal.GoFuzz {
+		zr.updateChecksum(zr.rd.BitsRead()-32, 0) // Record offset only
+	}
+	if zr.rd.ReadBitsBE64(1) != 0 {
+		panicf(errors.Deprecated, "block randomization is not supported")
+	}
+
+	// Read BWT related fields.
+	ptr := int(zr.rd.ReadBitsBE64(24)) // BWT origin pointer
+
+	// Read MTF related fields.
+	var dictArr [256]uint8
+	dict := dictArr[:0]
+	bmapHi := uint16(zr.rd.ReadBits(16))
+	for i := 0; i < 256; i, bmapHi = i+16, bmapHi>>1 {
+		if bmapHi&1 > 0 {
+			bmapLo := uint16(zr.rd.ReadBits(16))
+			for j := 0; j < 16; j, bmapLo = j+1, bmapLo>>1 {
+				if bmapLo&1 > 0 {
+					dict = append(dict, uint8(i+j))
+				}
+			}
+		}
+	}
+
+	// Step 1: Prefix encoding.
+	syms := zr.decodePrefix(len(dict))
+
+	// Step 2: Move-to-front transform and run-length encoding.
+	zr.mtf.Init(dict, zr.level*blockSize)
+	buf := zr.mtf.Decode(syms)
+
+	// Step 3: Burrows-Wheeler transformation.
+	if ptr >= len(buf) {
+		panicf(errors.Corrupted, "origin pointer (0x%06x) exceeds block size: %d", ptr, len(buf))
+	}
+	zr.bwt.Decode(buf, ptr)
+
+	return buf
+}
+
+func (zr *Reader) decodePrefix(numSyms int) (syms []uint16) {
+	numSyms += 2 // Remove 0 symbol, add RUNA, RUNB, and EOF symbols
+	if numSyms < 3 {
+		panicf(errors.Corrupted, "not enough prefix symbols: %d", numSyms)
+	}
+
+	// Read information about the trees and tree selectors.
+	var mtf internal.MoveToFront
+	numTrees := int(zr.rd.ReadBitsBE64(3))
+	if numTrees < minNumTrees || numTrees > maxNumTrees {
+		panicf(errors.Corrupted, "invalid number of prefix trees: %d", numTrees)
+	}
+	numSels := int(zr.rd.ReadBitsBE64(15))
+	if cap(zr.treeSels) < numSels {
+		zr.treeSels = make([]uint8, numSels)
+	}
+	treeSels := zr.treeSels[:numSels]
+	for i := range treeSels {
+		sym, ok := zr.rd.TryReadSymbol(&decSel)
+		if !ok {
+			sym = zr.rd.ReadSymbol(&decSel)
+		}
+		if int(sym) >= numTrees {
+			panicf(errors.Corrupted, "invalid prefix tree selector: %d", sym)
+		}
+		treeSels[i] = uint8(sym)
+	}
+	mtf.Decode(treeSels)
+	zr.treeSels = treeSels
+
+	// Initialize prefix codes.
+	for i := range zr.codes2D[:numTrees] {
+		zr.codes1D[i] = zr.codes2D[i][:numSyms]
+	}
+	zr.rd.ReadPrefixCodes(zr.codes1D[:numTrees], zr.trees1D[:numTrees])
+
+	// Read prefix encoded symbols of compressed data.
+	var tree *prefix.Decoder
+	var blkLen, selIdx int
+	syms = zr.syms[:0]
+	for {
+		if blkLen == 0 {
+			blkLen = numBlockSyms
+			if selIdx >= len(treeSels) {
+				panicf(errors.Corrupted, "not enough prefix tree selectors")
+			}
+			tree = &zr.trees1D[treeSels[selIdx]]
+			selIdx++
+		}
+		blkLen--
+		sym, ok := zr.rd.TryReadSymbol(tree)
+		if !ok {
+			sym = zr.rd.ReadSymbol(tree)
+		}
+
+		if int(sym) == numSyms-1 {
+			break // EOF marker
+		}
+		if int(sym) >= numSyms {
+			panicf(errors.Corrupted, "invalid prefix symbol: %d", sym)
+		}
+		if len(syms) >= zr.level*blockSize {
+			panicf(errors.Corrupted, "number of prefix symbols exceeds block size")
+		}
+		syms = append(syms, uint16(sym))
+	}
+	zr.syms = syms
+	return syms
+}
@@ -0,0 +1,101 @@
+// Copyright 2015, Joe Tsai. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE.md file.
+
+package bzip2
+
+import "github.com/dsnet/compress/internal/errors"
+
+// rleDone is a special "error" to indicate that the RLE stage is done.
+var rleDone = errorf(errors.Unknown, "RLE1 stage is completed")
+
+// runLengthEncoding implements the first RLE stage of bzip2. Every sequence
+// of 4..255 duplicated bytes is replaced by only the first 4 bytes, and a
+// single byte representing the repeat length. Similar to the C bzip2
+// implementation, the encoder will always terminate repeat sequences with a
+// count (even if it is the end of the buffer), and it will also never produce
+// run lengths of 256..259. The decoder can handle the latter case.
+//
+// For example, if the input was:
+//	input:  "AAAAAAABBBBCCCD"
+//
+// Then the output will be:
+//	output: "AAAA\x03BBBB\x00CCCD"
+type runLengthEncoding struct {
+	buf     []byte
+	idx     int
+	lastVal byte
+	lastCnt int
+}
+
+func (rle *runLengthEncoding) Init(buf []byte) {
+	*rle = runLengthEncoding{buf: buf}
+}
+
+func (rle *runLengthEncoding) Write(buf []byte) (int, error) {
+	for i, b := range buf {
+		if rle.lastVal != b {
+			rle.lastCnt = 0
+		}
+		rle.lastCnt++
+		switch {
+		case rle.lastCnt < 4:
+			if rle.idx >= len(rle.buf) {
+				return i, rleDone
+			}
+			rle.buf[rle.idx] = b
+			rle.idx++
+		case rle.lastCnt == 4:
+			if rle.idx+1 >= len(rle.buf) {
+				return i, rleDone
+			}
+			rle.buf[rle.idx] = b
+			rle.idx++
+			rle.buf[rle.idx] = 0
+			rle.idx++
+		case rle.lastCnt < 256:
+			rle.buf[rle.idx-1]++
+		default:
+			if rle.idx >= len(rle.buf) {
+				return i, rleDone
+			}
+			rle.lastCnt = 1
+			rle.buf[rle.idx] = b
+			rle.idx++
+		}
+		rle.lastVal = b
+	}
+	return len(buf), nil
+}
+
+func (rle *runLengthEncoding) Read(buf []byte) (int, error) {
+	for i := range buf {
+		switch {
+		case rle.lastCnt == -4:
+			if rle.idx >= len(rle.buf) {
+				return i, errorf(errors.Corrupted, "missing terminating run-length repeater")
+			}
+			rle.lastCnt = int(rle.buf[rle.idx])
+			rle.idx++
+			if rle.lastCnt > 0 {
+				break // Break the switch
+			}
+			fallthrough // Count was zero, continue the work
+		case rle.lastCnt <= 0:
+			if rle.idx >= len(rle.buf) {
+				return i, rleDone
+			}
+			b := rle.buf[rle.idx]
+			rle.idx++
+			if b != rle.lastVal {
+				rle.lastCnt = 0
+				rle.lastVal = b
+			}
+		}
+		buf[i] = rle.lastVal
+		rle.lastCnt--
+	}
+	return len(buf), nil
+}
+
+func (rle *runLengthEncoding) Bytes() []byte { return rle.buf[:rle.idx] }
@@ -0,0 +1,10 @@
+module github.com/dsnet/compress
+
+go 1.9
+
+require (
+	github.com/dsnet/golib v0.0.0-20171103203638-1ea166775780
+	github.com/klauspost/compress v1.4.1
+	github.com/klauspost/cpuid v1.2.0 // indirect
+	github.com/ulikunitz/xz v0.5.6
+)
@@ -0,0 +1,8 @@
+github.com/dsnet/golib v0.0.0-20171103203638-1ea166775780 h1:tFh1tRc4CA31yP6qDcu+Trax5wW5GuMxvkIba07qVLY=
+github.com/dsnet/golib v0.0.0-20171103203638-1ea166775780/go.mod h1:Lj+Z9rebOhdfkVLjJ8T6VcRQv3SXugXy999NBtR9aFY=
+github.com/klauspost/compress v1.4.1 h1:8VMb5+0wMgdBykOV96DwNwKFQ+WTI4pzYURP99CcB9E=
+github.com/klauspost/compress v1.4.1/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A=
+github.com/klauspost/cpuid v1.2.0 h1:NMpwD2G9JSFOE1/TJjGSo5zG7Yb2bTe7eq1jH+irmeE=
+github.com/klauspost/cpuid v1.2.0/go.mod h1:Pj4uuM528wm8OyEC2QMXAi2YiTZ96dNQPGgoMS4s3ek=
+github.com/ulikunitz/xz v0.5.6 h1:jGHAfXawEGZQ3blwU5wnWKQJvAraT7Ftq9EXjnXYgt8=
+github.com/ulikunitz/xz v0.5.6/go.mod h1:2bypXElzHzzJZwzH67Y6wb67pO62Rzfn7BSiF4ABRW8=
@@ -0,0 +1,107 @@
+// Copyright 2015, Joe Tsai. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE.md file.
+
+// Package internal is a collection of common compression algorithms.
+//
+// For performance reasons, these packages lack strong error checking and
+// require that the caller to ensure that strict invariants are kept.
+package internal
+
+var (
+	// IdentityLUT returns the input key itself.
+	IdentityLUT = func() (lut [256]byte) {
+		for i := range lut {
+			lut[i] = uint8(i)
+		}
+		return lut
+	}()
+
+	// ReverseLUT returns the input key with its bits reversed.
+	ReverseLUT = func() (lut [256]byte) {
+		for i := range lut {
+			b := uint8(i)
+			b = (b&0xaa)>>1 | (b&0x55)<<1
+			b = (b&0xcc)>>2 | (b&0x33)<<2
+			b = (b&0xf0)>>4 | (b&0x0f)<<4
+			lut[i] = b
+		}
+		return lut
+	}()
+)
+
+// ReverseUint32 reverses all bits of v.
+func ReverseUint32(v uint32) (x uint32) {
+	x |= uint32(ReverseLUT[byte(v>>0)]) << 24
+	x |= uint32(ReverseLUT[byte(v>>8)]) << 16
+	x |= uint32(ReverseLUT[byte(v>>16)]) << 8
+	x |= uint32(ReverseLUT[byte(v>>24)]) << 0
+	return x
+}
+
+// ReverseUint32N reverses the lower n bits of v.
+func ReverseUint32N(v uint32, n uint) (x uint32) {
+	return ReverseUint32(v << (32 - n))
+}
+
+// ReverseUint64 reverses all bits of v.
+func ReverseUint64(v uint64) (x uint64) {
+	x |= uint64(ReverseLUT[byte(v>>0)]) << 56
+	x |= uint64(ReverseLUT[byte(v>>8)]) << 48
+	x |= uint64(ReverseLUT[byte(v>>16)]) << 40
+	x |= uint64(ReverseLUT[byte(v>>24)]) << 32
+	x |= uint64(ReverseLUT[byte(v>>32)]) << 24
+	x |= uint64(ReverseLUT[byte(v>>40)]) << 16
+	x |= uint64(ReverseLUT[byte(v>>48)]) << 8
+	x |= uint64(ReverseLUT[byte(v>>56)]) << 0
+	return x
+}
+
+// ReverseUint64N reverses the lower n bits of v.
+func ReverseUint64N(v uint64, n uint) (x uint64) {
+	return ReverseUint64(v << (64 - n))
+}
+
+// MoveToFront is a data structure that allows for more efficient move-to-front
+// transformations. This specific implementation assumes that the alphabet is
+// densely packed within 0..255.
+type MoveToFront struct {
+	dict [256]uint8 // Mapping from indexes to values
+	tail int        // Number of tail bytes that are already ordered
+}
+
+func (m *MoveToFront) Encode(vals []uint8) {
+	copy(m.dict[:], IdentityLUT[:256-m.tail]) // Reset dict to be identity
+
+	var max int
+	for i, val := range vals {
+		var idx uint8 // Reverse lookup idx in dict
+		for di, dv := range m.dict {
+			if dv == val {
+				idx = uint8(di)
+				break
+			}
+		}
+		vals[i] = idx
+
+		max |= int(idx)
+		copy(m.dict[1:], m.dict[:idx])
+		m.dict[0] = val
+	}
+	m.tail = 256 - max - 1
+}
+
+func (m *MoveToFront) Decode(idxs []uint8) {
+	copy(m.dict[:], IdentityLUT[:256-m.tail]) // Reset dict to be identity
+
+	var max int
+	for i, idx := range idxs {
+		val := m.dict[idx] // Forward lookup val in dict
+		idxs[i] = val
+
+		max |= int(idx)
+		copy(m.dict[1:], m.dict[:idx])
+		m.dict[0] = val
+	}
+	m.tail = 256 - max - 1
+}
@@ -0,0 +1,12 @@
+// Copyright 2015, Joe Tsai. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE.md file.
+
+// +build debug,!gofuzz
+
+package internal
+
+const (
+	Debug  = true
+	GoFuzz = false
+)
@@ -0,0 +1,120 @@
+// Copyright 2016, Joe Tsai. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE.md file.
+
+// Package errors implements functions to manipulate compression errors.
+//
+// In idiomatic Go, it is an anti-pattern to use panics as a form of error
+// reporting in the API. Instead, the expected way to transmit errors is by
+// returning an error value. Unfortunately, the checking of "err != nil" in
+// tight loops commonly found in compression causes non-negligible performance
+// degradation. While this may not be idiomatic, the internal packages of this
+// repository rely on panics as a normal means to convey errors. In order to
+// ensure that these panics do not leak across the public API, the public
+// packages must recover from these panics and present an error value.
+//
+// The Panic and Recover functions in this package provide a safe way to
+// recover from errors only generated from within this repository.
+//
+// Example usage:
+//	func Foo() (err error) {
+//		defer errors.Recover(&err)
+//
+//		if rand.Intn(2) == 0 {
+//			// Unexpected panics will not be caught by Recover.
+//			io.Closer(nil).Close()
+//		} else {
+//			// Errors thrown by Panic will be caught by Recover.
+//			errors.Panic(errors.New("whoopsie"))
+//		}
+//	}
+//
+package errors
+
+import "strings"
+
+const (
+	// Unknown indicates that there is no classification for this error.
+	Unknown = iota
+
+	// Internal indicates that this error is due to an internal bug.
+	// Users should file a issue report if this type of error is encountered.
+	Internal
+
+	// Invalid indicates that this error is due to the user misusing the API
+	// and is indicative of a bug on the user's part.
+	Invalid
+
+	// Deprecated indicates the use of a deprecated and unsupported feature.
+	Deprecated
+
+	// Corrupted indicates that the input stream is corrupted.
+	Corrupted
+
+	// Closed indicates that the handlers are closed.
+	Closed
+)
+
+var codeMap = map[int]string{
+	Unknown:    "unknown error",
+	Internal:   "internal error",
+	Invalid:    "invalid argument",
+	Deprecated: "deprecated format",
+	Corrupted:  "corrupted input",
+	Closed:     "closed handler",
+}
+
+type Error struct {
+	Code int    // The error type
+	Pkg  string // Name of the package where the error originated
+	Msg  string // Descriptive message about the error (optional)
+}
+
+func (e Error) Error() string {
+	var ss []string
+	for _, s := range []string{e.Pkg, codeMap[e.Code], e.Msg} {
+		if s != "" {
+			ss = append(ss, s)
+		}
+	}
+	return strings.Join(ss, ": ")
+}
+
+func (e Error) CompressError()     {}
+func (e Error) IsInternal() bool   { return e.Code == Internal }
+func (e Error) IsInvalid() bool    { return e.Code == Invalid }
+func (e Error) IsDeprecated() bool { return e.Code == Deprecated }
+func (e Error) IsCorrupted() bool  { return e.Code == Corrupted }
+func (e Error) IsClosed() bool     { return e.Code == Closed }
+
+func IsInternal(err error) bool   { return isCode(err, Internal) }
+func IsInvalid(err error) bool    { return isCode(err, Invalid) }
+func IsDeprecated(err error) bool { return isCode(err, Deprecated) }
+func IsCorrupted(err error) bool  { return isCode(err, Corrupted) }
+func IsClosed(err error) bool     { return isCode(err, Closed) }
+
+func isCode(err error, code int) bool {
+	if cerr, ok := err.(Error); ok && cerr.Code == code {
+		return true
+	}
+	return false
+}
+
+// errWrap is used by Panic and Recover to ensure that only errors raised by
+// Panic are recovered by Recover.
+type errWrap struct{ e *error }
+
+func Recover(err *error) {
+	switch ex := recover().(type) {
+	case nil:
+		// Do nothing.
+	case errWrap:
+		*err = *ex.e
+	default:
+		panic(ex)
+	}
+}
+
+func Panic(err error) {
+	panic(errWrap{&err})
+}
@@ -0,0 +1,12 @@
+// Copyright 2016, Joe Tsai. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE.md file.
+
+// +build gofuzz
+
+package internal
+
+const (
+	Debug  = true
+	GoFuzz = true
+)
@@ -0,0 +1,159 @@
+// Copyright 2015, Joe Tsai. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE.md file.
+
+// +build debug
+
+package prefix
+
+import (
+	"fmt"
+	"math"
+	"strings"
+)
+
+func max(a, b int) int {
+	if a > b {
+		return a
+	}
+	return b
+}
+
+func lenBase2(n uint) int {
+	return int(math.Ceil(math.Log2(float64(n + 1))))
+}
+func padBase2(v, n uint, m int) string {
+	s := fmt.Sprintf("%b", 1<<n|v)[1:]
+	if pad := m - len(s); pad > 0 {
+		return strings.Repeat(" ", pad) + s
+	}
+	return s
+}
+
+func lenBase10(n int) int {
+	return int(math.Ceil(math.Log10(float64(n + 1))))
+}
+func padBase10(n, m int) string {
+	s := fmt.Sprintf("%d", n)
+	if pad := m - len(s); pad > 0 {
+		return strings.Repeat(" ", pad) + s
+	}
+	return s
+}
+
+func (rc RangeCodes) String() string {
+	var maxLen, maxBase int
+	for _, c := range rc {
+		maxLen = max(maxLen, int(c.Len))
+		maxBase = max(maxBase, int(c.Base))
+	}
+
+	var ss []string
+	ss = append(ss, "{")
+	for i, c := range rc {
+		base := padBase10(int(c.Base), lenBase10(maxBase))
+		if c.Len > 0 {
+			base += fmt.Sprintf("-%d", c.End()-1)
+		}
+		ss = append(ss, fmt.Sprintf("\t%s:  {len: %s, range: %s},",
+			padBase10(int(i), lenBase10(len(rc)-1)),
+			padBase10(int(c.Len), lenBase10(maxLen)),
+			base,
+		))
+	}
+	ss = append(ss, "}")
+	return strings.Join(ss, "\n")
+}
+
+func (pc PrefixCodes) String() string {
+	var maxSym, maxLen, maxCnt int
+	for _, c := range pc {
+		maxSym = max(maxSym, int(c.Sym))
+		maxLen = max(maxLen, int(c.Len))
+		maxCnt = max(maxCnt, int(c.Cnt))
+	}
+
+	var ss []string
+	ss = append(ss, "{")
+	for _, c := range pc {
+		var cntStr string
+		if maxCnt > 0 {
+			cnt := int(32*float32(c.Cnt)/float32(maxCnt) + 0.5)
+			cntStr = fmt.Sprintf("%s |%s",
+				padBase10(int(c.Cnt), lenBase10(maxCnt)),
+				strings.Repeat("#", cnt),
+			)
+		}
+		ss = append(ss, fmt.Sprintf("\t%s:  %s,  %s",
+			padBase10(int(c.Sym), lenBase10(maxSym)),
+			padBase2(uint(c.Val), uint(c.Len), maxLen),
+			cntStr,
+		))
+	}
+	ss = append(ss, "}")
+	return strings.Join(ss, "\n")
+}
+
+func (pd Decoder) String() string {
+	var ss []string
+	ss = append(ss, "{")
+	if len(pd.chunks) > 0 {
+		ss = append(ss, "\tchunks: {")
+		for i, c := range pd.chunks {
+			label := "sym"
+			if uint(c&countMask) > uint(pd.chunkBits) {
+				label = "idx"
+			}
+			ss = append(ss, fmt.Sprintf("\t\t%s:  {%s: %s, len: %s}",
+				padBase2(uint(i), uint(pd.chunkBits), int(pd.chunkBits)),
+				label, padBase10(int(c>>countBits), 3),
+				padBase10(int(c&countMask), 2),
+			))
+		}
+		ss = append(ss, "\t},")
+
+		for j, links := range pd.links {
+			ss = append(ss, fmt.Sprintf("\tlinks[%d]: {", j))
+			linkBits := lenBase2(uint(pd.linkMask))
+			for i, c := range links {
+				ss = append(ss, fmt.Sprintf("\t\t%s:  {sym: %s, len: %s},",
+					padBase2(uint(i), uint(linkBits), int(linkBits)),
+					padBase10(int(c>>countBits), 3),
+					padBase10(int(c&countMask), 2),
+				))
+			}
+			ss = append(ss, "\t},")
+		}
+	}
+	ss = append(ss, fmt.Sprintf("\tchunkMask: %b,", pd.chunkMask))
+	ss = append(ss, fmt.Sprintf("\tlinkMask:  %b,", pd.linkMask))
+	ss = append(ss, fmt.Sprintf("\tchunkBits: %d,", pd.chunkBits))
+	ss = append(ss, fmt.Sprintf("\tMinBits:   %d,", pd.MinBits))
+	ss = append(ss, fmt.Sprintf("\tNumSyms:   %d,", pd.NumSyms))
+	ss = append(ss, "}")
+	return strings.Join(ss, "\n")
+}
+
+func (pe Encoder) String() string {
+	var maxLen int
+	for _, c := range pe.chunks {
+		maxLen = max(maxLen, int(c&countMask))
+	}
+
+	var ss []string
+	ss = append(ss, "{")
+	if len(pe.chunks) > 0 {
+		ss = append(ss, "\tchunks: {")
+		for i, c := range pe.chunks {
+			ss = append(ss, fmt.Sprintf("\t\t%s:  %s,",
+				padBase10(i, 3),
+				padBase2(uint(c>>countBits), uint(c&countMask), maxLen),
+			))
+		}
+		ss = append(ss, "\t},")
+	}
+	ss = append(ss, fmt.Sprintf("\tchunkMask: %b,", pe.chunkMask))
+	ss = append(ss, fmt.Sprintf("\tNumSyms:   %d,", pe.NumSyms))
+	ss = append(ss, "}")
+	return strings.Join(ss, "\n")
+}
@@ -0,0 +1,136 @@
+// Copyright 2015, Joe Tsai. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE.md file.
+
+package prefix
+
+import (
+	"sort"
+
+	"github.com/dsnet/compress/internal"
+)
+
+// The algorithm used to decode variable length codes is based on the lookup
+// method in zlib. If the code is less-than-or-equal to maxChunkBits,
+// then the symbol can be decoded using a single lookup into the chunks table.
+// Otherwise, the links table will be used for a second level lookup.
+//
+// The chunks slice is keyed by the contents of the bit buffer ANDed with
+// the chunkMask to avoid a out-of-bounds lookup. The value of chunks is a tuple
+// that is decoded as follow:
+//
+//	var length = chunks[bitBuffer&chunkMask] & countMask
+//	var symbol = chunks[bitBuffer&chunkMask] >> countBits
+//
+// If the decoded length is larger than chunkBits, then an overflow link table
+// must be used for further decoding. In this case, the symbol is actually the
+// index into the links tables. The second-level links table returned is
+// processed in the same way as the chunks table.
+//
+//	if length > chunkBits {
+//		var index = symbol // Previous symbol is index into links tables
+//		length = links[index][bitBuffer>>chunkBits & linkMask] & countMask
+//		symbol = links[index][bitBuffer>>chunkBits & linkMask] >> countBits
+//	}
+//
+// See the following:
+//	http://www.gzip.org/algorithm.txt
+
+type Decoder struct {
+	chunks    []uint32   // First-level lookup map
+	links     [][]uint32 // Second-level lookup map
+	chunkMask uint32     // Mask the length of the chunks table
+	linkMask  uint32     // Mask the length of the link table
+	chunkBits uint32     // Bit-length of the chunks table
+
+	MinBits uint32 // The minimum number of bits to safely make progress
+	NumSyms uint32 // Number of symbols
+}
+
+// Init initializes Decoder according to the codes provided.
+func (pd *Decoder) Init(codes PrefixCodes) {
+	// Handle special case trees.
+	if len(codes) <= 1 {
+		switch {
+		case len(codes) == 0: // Empty tree (should error if used later)
+			*pd = Decoder{chunks: pd.chunks[:0], links: pd.links[:0], NumSyms: 0}
+		case len(codes) == 1 && codes[0].Len == 0: // Single code tree (bit-length of zero)
+			pd.chunks = append(pd.chunks[:0], codes[0].Sym<<countBits|0)
+			*pd = Decoder{chunks: pd.chunks[:1], links: pd.links[:0], NumSyms: 1}
+		default:
+			panic("invalid codes")
+		}
+		return
+	}
+	if internal.Debug && !sort.IsSorted(prefixCodesBySymbol(codes)) {
+		panic("input codes is not sorted")
+	}
+	if internal.Debug && !(codes.checkLengths() && codes.checkPrefixes()) {
+		panic("detected incomplete or overlapping codes")
+	}
+
+	var minBits, maxBits uint32 = valueBits, 0
+	for _, c := range codes {
+		if minBits > c.Len {
+			minBits = c.Len
+		}
+		if maxBits < c.Len {
+			maxBits = c.Len
+		}
+	}
+
+	// Allocate chunks table as needed.
+	const maxChunkBits = 9 // This can be tuned for better performance
+	pd.NumSyms = uint32(len(codes))
+	pd.MinBits = minBits
+	pd.chunkBits = maxBits
+	if pd.chunkBits > maxChunkBits {
+		pd.chunkBits = maxChunkBits
+	}
+	numChunks := 1 << pd.chunkBits
+	pd.chunks = allocUint32s(pd.chunks, numChunks)
+	pd.chunkMask = uint32(numChunks - 1)
+
+	// Allocate links tables as needed.
+	pd.links = pd.links[:0]
+	pd.linkMask = 0
+	if pd.chunkBits < maxBits {
+		numLinks := 1 << (maxBits - pd.chunkBits)
+		pd.linkMask = uint32(numLinks - 1)
+
+		var linkIdx uint32
+		for i := range pd.chunks {
+			pd.chunks[i] = 0 // Logic below relies on zero value as uninitialized
+		}
+		for _, c := range codes {
+			if c.Len > pd.chunkBits && pd.chunks[c.Val&pd.chunkMask] == 0 {
+				pd.chunks[c.Val&pd.chunkMask] = (linkIdx << countBits) | (pd.chunkBits + 1)
+				linkIdx++
+			}
+		}
+
+		pd.links = extendSliceUint32s(pd.links, int(linkIdx))
+		linksFlat := allocUint32s(pd.links[0], numLinks*int(linkIdx))
+		for i, j := 0, 0; i < len(pd.links); i, j = i+1, j+numLinks {
+			pd.links[i] = linksFlat[j : j+numLinks]
+		}
+	}
+
+	// Fill out chunks and links tables with values.
+	for _, c := range codes {
+		chunk := c.Sym<<countBits | c.Len
+		if c.Len <= pd.chunkBits {
+			skip := 1 << uint(c.Len)
+			for j := int(c.Val); j < len(pd.chunks); j += skip {
+				pd.chunks[j] = chunk
+			}
+		} else {
+			linkIdx := pd.chunks[c.Val&pd.chunkMask] >> countBits
+			links := pd.links[linkIdx]
+			skip := 1 << uint(c.Len-pd.chunkBits)
+			for j := int(c.Val >> pd.chunkBits); j < len(links); j += skip {
+				links[j] = chunk
+			}
+		}
+	}
+}
@@ -0,0 +1,66 @@
+// Copyright 2015, Joe Tsai. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE.md file.
+
+package prefix
+
+import (
+	"sort"
+
+	"github.com/dsnet/compress/internal"
+)
+
+type Encoder struct {
+	chunks    []uint32 // First-level lookup map
+	chunkMask uint32   // Mask the length of the chunks table
+
+	NumSyms uint32 // Number of symbols
+}
+
+// Init initializes Encoder according to the codes provided.
+func (pe *Encoder) Init(codes PrefixCodes) {
+	// Handle special case trees.
+	if len(codes) <= 1 {
+		switch {
+		case len(codes) == 0: // Empty tree (should error if used later)
+			*pe = Encoder{chunks: pe.chunks[:0], NumSyms: 0}
+		case len(codes) == 1 && codes[0].Len == 0: // Single code tree (bit-length of zero)
+			pe.chunks = append(pe.chunks[:0], codes[0].Val<<countBits|0)
+			*pe = Encoder{chunks: pe.chunks[:1], NumSyms: 1}
+		default:
+			panic("invalid codes")
+		}
+		return
+	}
+	if internal.Debug && !sort.IsSorted(prefixCodesBySymbol(codes)) {
+		panic("input codes is not sorted")
+	}
+	if internal.Debug && !(codes.checkLengths() && codes.checkPrefixes()) {
+		panic("detected incomplete or overlapping codes")
+	}
+
+	// Enough chunks to contain all the symbols.
+	numChunks := 1
+	for n := len(codes) - 1; n > 0; n >>= 1 {
+		numChunks <<= 1
+	}
+	pe.NumSyms = uint32(len(codes))
+
+retry:
+	// Allocate and reset chunks.
+	pe.chunks = allocUint32s(pe.chunks, numChunks)
+	pe.chunkMask = uint32(numChunks - 1)
+	for i := range pe.chunks {
+		pe.chunks[i] = 0 // Logic below relies on zero value as uninitialized
+	}
+
+	// Insert each symbol, checking that there are no conflicts.
+	for _, c := range codes {
+		if pe.chunks[c.Sym&pe.chunkMask] > 0 {
+			// Collision found our "hash" table, so grow and try again.
+			numChunks <<= 1
+			goto retry
+		}
+		pe.chunks[c.Sym&pe.chunkMask] = c.Val<<countBits | c.Len
+	}
+}
--- a/Show More
+++ b/Show More