Fixes 1693: support .xz and .zst compression types (#14)

go.mod

@@ -5,7 +5,10 @@ go 1.16
 require (
 	github.com/ProtonMail/go-crypto v0.0.0-20230124153114-0acdc8ae009b
 	github.com/cloudflare/circl v1.3.3 // indirect
+	github.com/h2non/filetype v1.1.3
+	github.com/klauspost/compress v1.16.5
 	github.com/openlyinc/pointy v1.2.0
 	github.com/stretchr/testify v1.8.1
+	github.com/ulikunitz/xz v0.5.11
 	golang.org/x/crypto v0.5.0 // indirect
 )

go.sum

@@ -8,6 +8,10 @@ github.com/cloudflare/circl v1.3.3/go.mod h1:5XYMA4rFBvNIrhs50XuiBJ15vF2pZn4nnUK
 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
 github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/h2non/filetype v1.1.3 h1:FKkx9QbD7HR/zjK1Ia5XiBsq9zdLi5Kf3zGyFTAFkGg=
+github.com/h2non/filetype v1.1.3/go.mod h1:319b3zT68BvV+WRj7cwy856M2ehB3HqNOt6sy1HndBY=
+github.com/klauspost/compress v1.16.5 h1:IFV2oUNUzZaz+XyusxpLzpzS8Pt5rh0Z16For/djlyI=
+github.com/klauspost/compress v1.16.5/go.mod h1:ntbaceVETuRiXiv4DpjP66DpAtAGkEQskQzEyD//IeE=
 github.com/openlyinc/pointy v1.2.0 h1:vbb/WoPbshyTH8j3/XYu3enlZfv+NHxAD15qTm1zbk0=
 github.com/openlyinc/pointy v1.2.0/go.mod h1:JodZOTJoBNaAQHeU0F/SwA4PL0lg4pKF7fYFpX291P0=
 github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
@@ -20,6 +24,8 @@ github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/
 github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU=
 github.com/stretchr/testify v1.8.1 h1:w7B6lhMri9wdJUVmEZPGGhZzrYTPvgJArz7wNPgYKsk=
 github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4=
+github.com/ulikunitz/xz v0.5.11 h1:kpFauv27b6ynzBNT/Xy+1k+fK4WswhN/6PN5WhFAGw8=
+github.com/ulikunitz/xz v0.5.11/go.mod h1:nbz6k7qbPmH4IRqmfOplQw/tblSgqTqBwxkY0oWt/14=
 github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY=
 golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
 golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=

pkg/yum/repository.go

@@ -1,6 +1,7 @@
 package yum
 
 import (
+	"bytes"
 	"compress/gzip"
 	"encoding/xml"
 	"fmt"
@@ -9,6 +10,11 @@ import (
 	"net/http"
 	"net/url"
 	"path"
+
+	"github.com/h2non/filetype"
+	"github.com/h2non/filetype/matchers"
+	"github.com/klauspost/compress/zstd"
+	"github.com/ulikunitz/xz"
 )
 
 // Package metadata of a given package
@@ -165,7 +171,7 @@ func (r *Repository) Packages() ([]Package, int, error) {
 		return nil, resp.StatusCode, fmt.Errorf("Cannot fetch %v: %d", primaryURL, resp.StatusCode)
 	}
 
-	if packages, err = ParseCompressedXMLData(resp.Body); err != nil {
+	if packages, err = ParseCompressedXMLData(io.NopCloser(resp.Body)); err != nil {
 		return nil, resp.StatusCode, err
 	}
 	r.packages = packages
@@ -273,17 +279,38 @@ func ParseRepomdXML(body io.ReadCloser) (Repomd, error) {
 	return result, err
 }
 
-// Unzips a gzipped body response, then parses the contained XML for package information
+// Unzips a compressed body response, then parses the contained XML for package information
 // Returns an array of package data
-func ParseCompressedXMLData(body io.ReadCloser) ([]Package, error) {
-	reader, err := gzip.NewReader(body)
+func ParseCompressedXMLData(body io.Reader) ([]Package, error) {
+	var reader, data io.Reader
+	var err error
+
+	data = body
+	readData, err := io.ReadAll(data)
+	if err != nil {
+		return []Package{}, err
+	}
+	fileType, err := filetype.Match(readData)
+	if err != nil {
+		return []Package{}, err
+	}
 
+	data = bytes.NewReader(readData)
+	switch fileType {
+	case matchers.TypeGz:
+		reader, err = gzip.NewReader(data)
+	case matchers.TypeZstd:
+		reader, err = zstd.NewReader(data)
+	case matchers.TypeXz:
+		reader, err = xz.NewReader(data)
+	default:
+		return []Package{}, fmt.Errorf("invalid file type: must be gzip, xz, or zstd.")
+	}
 	if err != nil {
 		return []Package{}, fmt.Errorf("Error unzipping response body: %w", err)
 	}
 
 	decoder := xml.NewDecoder(reader)
-	reader.Close()
 
 	result := []Package{}
 

pkg/yum/repository_test.go

@@ -187,28 +187,36 @@ func TestFetchRepomdSignature(t *testing.T) {
 	assert.Nil(t, err)
 }
 
-// Check that the parser can decompress a gzip file and read the correct number of packages
+// Check that the parser can decompress a compressed file and read the correct number of packages
 func TestParseCompressedXMLData(t *testing.T) {
-	xmlFile, err := os.Open("mocks/primary.xml.gz")
-	if err != nil {
-		log.Fatal(err)
+	paths := []string{
+		"mocks/primary.xml.gz",
+		"mocks/primary.xml.xz",
+		"mocks/primary.xml.zst",
 	}
-	defer xmlFile.Close()
-	result, err := ParseCompressedXMLData(xmlFile)
-	if err != nil {
-		t.Errorf("Error in test: %v", err)
-	}
-	if len(result) != 2 {
-		t.Errorf("Error - Expected to return 2 packages but received: %v", len(result))
-	}
-	if result[0].Checksum.Type != "sha1" {
-		t.Errorf(fmt.Sprintf("Checksum of %s received, should be sha1", result[0].Checksum.Type))
-	}
-	if result[0].Summary == "" {
-		t.Errorf("Did not properly parse summary")
-	}
-	if result[0].Checksum.Value == "" {
-		t.Errorf("Did not properly parse checksum")
+
+	for _, path := range paths {
+		xmlFile, err := os.Open(path)
+		if err != nil {
+			log.Fatal(err)
+		}
+		defer xmlFile.Close()
+		result, err := ParseCompressedXMLData(xmlFile)
+		if err != nil {
+			t.Errorf("Error in test: %v", err)
+		}
+		if len(result) != 2 {
+			t.Errorf("Error - Expected to return 2 packages but received: %v", len(result))
+		}
+		if result[0].Checksum.Type != "sha1" {
+			t.Errorf(fmt.Sprintf("Checksum of %s received, should be sha1", result[0].Checksum.Type))
+		}
+		if result[0].Summary == "" {
+			t.Errorf("Did not properly parse summary")
+		}
+		if result[0].Checksum.Value == "" {
+			t.Errorf("Did not properly parse checksum")
+		}
 	}
 }