From d18876aa7799db1a61dcc96d6163b185e00a4024 Mon Sep 17 00:00:00 2001
From: rverdile <ryanverdile@gmail.com>
Date: Wed, 17 May 2023 14:57:20 -0400
Subject: [PATCH] Fixes 1693: support .xz and .zst compression types (#14)

---
 go.mod                        |   3 +++
 go.sum                        |   6 +++++
 pkg/yum/mocks/primary.xml.xz  | Bin 0 -> 1312 bytes
 pkg/yum/mocks/primary.xml.zst | Bin 0 -> 1369 bytes
 pkg/yum/repository.go         |  37 ++++++++++++++++++++++----
 pkg/yum/repository_test.go    |  48 ++++++++++++++++++++--------------
 6 files changed, 69 insertions(+), 25 deletions(-)
 create mode 100644 pkg/yum/mocks/primary.xml.xz
 create mode 100644 pkg/yum/mocks/primary.xml.zst

diff --git a/go.mod b/go.mod
index d585797..5e7898a 100644
--- a/go.mod
+++ b/go.mod
@@ -5,7 +5,10 @@ go 1.16
 require (
 	github.com/ProtonMail/go-crypto v0.0.0-20230124153114-0acdc8ae009b
 	github.com/cloudflare/circl v1.3.3 // indirect
+	github.com/h2non/filetype v1.1.3
+	github.com/klauspost/compress v1.16.5
 	github.com/openlyinc/pointy v1.2.0
 	github.com/stretchr/testify v1.8.1
+	github.com/ulikunitz/xz v0.5.11
 	golang.org/x/crypto v0.5.0 // indirect
 )
diff --git a/go.sum b/go.sum
index fa49f2c..794f7d5 100644
--- a/go.sum
+++ b/go.sum
@@ -8,6 +8,10 @@ github.com/cloudflare/circl v1.3.3/go.mod h1:5XYMA4rFBvNIrhs50XuiBJ15vF2pZn4nnUK
 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
 github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/h2non/filetype v1.1.3 h1:FKkx9QbD7HR/zjK1Ia5XiBsq9zdLi5Kf3zGyFTAFkGg=
+github.com/h2non/filetype v1.1.3/go.mod h1:319b3zT68BvV+WRj7cwy856M2ehB3HqNOt6sy1HndBY=
+github.com/klauspost/compress v1.16.5 h1:IFV2oUNUzZaz+XyusxpLzpzS8Pt5rh0Z16For/djlyI=
+github.com/klauspost/compress v1.16.5/go.mod h1:ntbaceVETuRiXiv4DpjP66DpAtAGkEQskQzEyD//IeE=
 github.com/openlyinc/pointy v1.2.0 h1:vbb/WoPbshyTH8j3/XYu3enlZfv+NHxAD15qTm1zbk0=
 github.com/openlyinc/pointy v1.2.0/go.mod h1:JodZOTJoBNaAQHeU0F/SwA4PL0lg4pKF7fYFpX291P0=
 github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
@@ -20,6 +24,8 @@ github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/
 github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU=
 github.com/stretchr/testify v1.8.1 h1:w7B6lhMri9wdJUVmEZPGGhZzrYTPvgJArz7wNPgYKsk=
 github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4=
+github.com/ulikunitz/xz v0.5.11 h1:kpFauv27b6ynzBNT/Xy+1k+fK4WswhN/6PN5WhFAGw8=
+github.com/ulikunitz/xz v0.5.11/go.mod h1:nbz6k7qbPmH4IRqmfOplQw/tblSgqTqBwxkY0oWt/14=
 github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY=
 golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
 golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
diff --git a/pkg/yum/mocks/primary.xml.xz b/pkg/yum/mocks/primary.xml.xz
new file mode 100644
index 0000000000000000000000000000000000000000..6c41c24bdb98beebd45becefb98476106459d278
GIT binary patch
literal 1312
zcmV+*1>gGpH+ooF000E$*0e?f03iVu0001VFXf})5V-{1T>u^r%ZCxz&SsGhgC5HL
zdo$F3>9^V_F?J<pAWbR>ZEE)^a5ymM%pF!2BFRpseW9)Nyl8l10`5XF_(P7pQT6NG
z4F2VTec)djRw5K7)ZZrQzJ#%93g1=#@<Qx<QzUO{FY@!bR!LqI^SAjgr46o-7iC9j
z8k$@&w2m(FA(w%ebZRpB=(o4^7>30(S{y|__U(0dW(3~Th%H7s<=syFV7vr}iKOx#
zB1>ls4dmEx0YVBe@4RHvy?OeE_on{PqH**p33hPA9&tz$dgu6xT6(j@C`I;~j{qJ-
zJc9q_J9Ok7)94o$fToj#ksOx;a#x40seuxRv7zEAVUiaIy>ho(0<>(W`2nGHf|vVu
zXP2CvjS&B2k>9N-$%p-^D|gSS$0+lygSjMEqOR4I**}?KEfRk8M--c-c@-#N5O{%j
z4n|FwKBugzZ#uvF40#!PZ@H{PeQBIas-QM%_&l{zcJa9_V~!at;{+avlsRue+u}76
zx<#4EsJZmit$grABis@}LXD^X{p59V3Qy6Lp<NV!=##2s)xN)9R0}gc6HJc$<^mfe
zQ4$84A1qE_Xq$}Rx7;<U-((Fs+1y5dN-DIy9M5rl=*+ZT9E43wD>wJB1qx@I;daw9
z(~%$$*+@4rrLUm1V9x}WpRF`IUQ!X8cM=#2T4BGKX9t^!7buAy_$c6tlwjlP`j1+7
z383T>p%?R?Y*DdFB4NxPpb|P;QJ)p&<L4wyHjf)9hdrlVqrLW-OmwcwN?4T7ilY$Q
z6yzp&w?1Fl&|MJ=<UB5MFSu$#ko~z??)Dz_DNsUjCy>L$H<>&0IOs5a$ARZ?4i$~4
z7L23mxMNA!VR>AoZy?%*B0|1Jbr)`pC51kXoRN0E{>d3YE9*;qpbAj*UNObE@yX>v
z&1c?QAc;PJ1>gvqrhbfiHnoSr>uhWfTaKGcGsBsSK)l|lVGd5ZPK9}0CH;ddI42{4
zt8oQizO=#SSOh8c@XNp^hEp6CsUhcwVlxlZw5B@lrI?^aRO~uYb5b2Uxa})>CCr*g
zVxDsVMeq>t$2k}=HvhG5KN$XoKzl2ge++jgAI%F(k$EeUr|1B_f-=4{d=awvXP`W0
zw*tWb)Dyqz+{w^+cq{&v`QVcu?9HhO@^o$$K~R(Mys>2&PAusg?Ih=qaZmf4aO~9c
z(r6-4l{J4|Sx#ok?lsvXigxeHhMBI!B{3Uyj;Xd10uQf67Zd!%DW_WJj{PnvO;5RF
zTWqEHLjKLZ7PI#-_f}p?mae=uXEO1VEap<ugQYq330$a*kCHj#N?>%`fB`|>wpW=3
zhA@3n=TZb04gs{ohND#k5~lBWYxI$W?99tdvA5Cqt9SY|eFGmWr9<9FLI*wME=n<l
zarlS_l1{-Wp^ELOp4);4DeX$86GoR9W5P~o{>jn`@i0yH*v`E2x^f(Z?w0^GT}bi5
zwV}~Rd42+0(d|zDm1t_I?fF@4njq}cPJ2_9tfJSzV82#gI1o|)NI#XYR3Y7!qm2Gf
zxg#2?r(RqyX?E8ZBsJuSK&ZLMxtObd2sCB+J1^6GUegpwNjpt$6qc9ec*r)0e8j3q
zi%#r_1M(?5{A<?oxuUy1rDAo+AF=Omw)D0hcfR^>4Iz>M0002bpw^ukXsCn$0r?5K
WApihWe3-Yf#Ao{g000001X)^;vwFAy

literal 0
HcmV?d00001

diff --git a/pkg/yum/mocks/primary.xml.zst b/pkg/yum/mocks/primary.xml.zst
new file mode 100644
index 0000000000000000000000000000000000000000..cc6db7255f5aec355312f969cec0e2c0d8761790
GIT binary patch
literal 1369
zcmV-f1*ZBawJ-f(x(|IS0QyHn4Jq)9Ru3@?#E2*{)I55=(^YYPHq!Z|&3ur5pu@!D
z%i%jkX}~iub~32P0LB2v0C<|Tb9L-m>JUfZh~S7}*}q7E;xaYv3RD!?kWjh<E6ACk
zX;Ku$VV;F)OcgUZ+7hpC5)2QTL`~~)8_WJ7{gSWS;-JMfO~MZ8)}Zd5rAgFrH>F9J
zwAVZ8UcAV8oawkEj#2mGMbsfqCI8joSgcWZll(?<M+-;8F2*hvP%Va8`Cdl3^G8Ox
zimXoho`9S4)ugW!tLU~c5Ysg2-HHzxZ^s;|m>r824g&2-5==N~l2s>VW%X`~Tbfku
zeC&{>SOtM77`9c^ja}@V$C)lkUVR_Yq{{Kt)y^EHr4a)yjD=}ZbC{*UXf%m(N@M;y
zyFCm8I*ledWL^@-_6hSLhXJ7m6$ok|1Hz?A6rI`Bj?GMQ_M-P+-+bKn@3Zs>7T;%K
zYcWSbLE)e{i@Fyt8jvjjn3XQai*oEDrE`LnO#?Co&`wqb9Uw=Cro4dzcQ0P#t~@E)
zs?$<*Rvme0s>a4nMR#yiJM*1bzEe^6;zjRrLa<A8Bu|`4m*@w!6xkOvwpG7R90Q9(
zvC&jgGLVAYQ5cG9oZ^-SIuM+v;;FSc@oX~*33l^~x)(3{uMVfPiWnO3{i?vwKxfYC
z!{*9<@?W0;!{e02L3|k+66<mCXV<?vr-x@I!EyJ(e^sz%RjsrhSI!*sFECBwJQWH}
zaxB(v3>cd9R-{z~=j!j&gjjsx(=1KOIOgxBFc31DL{)FMYQ)KPv6J|*i(TT({@ck}
z)tzfBO5!{fj{Np|csH}yvQ%}*ixkH|9wHzd{?+BA0<ept?(T&n$BvjI6|-Y6Qcdg-
z&yl)W&8V4Wj~O3W#2-|&^!j*Vd9zy(u$vtu3OGP;u%JPKq8TsFzMJ=R1G<Lc7{@?3
zU_ik`!$O`DbQBg89B1FjX;SDs@PRykMB@FV_ZC1PVZbHH%S_WihVd}q0-Xkei{db_
z0UfAepoW2u0yWsp*{iD_+iH=qy*K;r9IxDuMQmAoyRy<LwX$RX?5EaFFldN~i3J7h
z*LFihOtfFQtvWyJYCY%Jlz*HOh=_^F8PgM=rV*b;#)bmp<OG1tsBx7t2q`lYh{$M?
zf}{gLLSh)BVh0kaGKrETAw-ZOMo0)EL?)1kf)OE+)R+S^NCvov-ahGcM<RLPbg^_o
zCE0@_!|RVHZ#?9!T3MpiX&FmuSK!mm#&b!1hm7aq)6&)joHJVQAPLXf@Kw}aS`D&E
z#Hd22Ac(&Sf;14pCk#{SX}CdXozo%U7RUoG5Qhd{v<-*=<W28IBKv?`{Z-lYz-Wa8
z5rBP{3CLCz5I(<AgFCGrK?9@{X5f?qK0Nx`IzS}SN0-u)fD~XGkQO=@_r1M3)Dynr
zD-3^uZU=)5$G(moCLU)WedD%sVClOv>bsAlZ))*PBQ)ni<~5UID}4YNVl41obw^IA
zd<cMM@K7}>yj{N=&d#QFBxcY~u-QWNH@>$881g#LYb7RGJk^sIk%)?X#))<F2l<7=
z07}TjJ=gmMVkvL*Ua?sLv^HHU^7Xh0#6}DLJm5+verS$LE!<xtcbKF(YAca8aUncm
zZh<}zRDv;}L!Gyf4GA!__7tO#Dpevy{ooy*1-1+B^=ArsN9v1E+~_bAl}~`+r=WK|
ziX4;=z6C#YNpWoa>yZ{d@EOQ(m(w9Lz72KUn}!YQTdJyM&Ll<Xi4J+DX#?sOCcq)9
b3~#d7C-KpC<vde_T%Kv?s#e51pGK)FST&71

literal 0
HcmV?d00001

diff --git a/pkg/yum/repository.go b/pkg/yum/repository.go
index d3986b3..824d025 100644
--- a/pkg/yum/repository.go
+++ b/pkg/yum/repository.go
@@ -1,6 +1,7 @@
 package yum
 
 import (
+	"bytes"
 	"compress/gzip"
 	"encoding/xml"
 	"fmt"
@@ -9,6 +10,11 @@ import (
 	"net/http"
 	"net/url"
 	"path"
+
+	"github.com/h2non/filetype"
+	"github.com/h2non/filetype/matchers"
+	"github.com/klauspost/compress/zstd"
+	"github.com/ulikunitz/xz"
 )
 
 // Package metadata of a given package
@@ -165,7 +171,7 @@ func (r *Repository) Packages() ([]Package, int, error) {
 		return nil, resp.StatusCode, fmt.Errorf("Cannot fetch %v: %d", primaryURL, resp.StatusCode)
 	}
 
-	if packages, err = ParseCompressedXMLData(resp.Body); err != nil {
+	if packages, err = ParseCompressedXMLData(io.NopCloser(resp.Body)); err != nil {
 		return nil, resp.StatusCode, err
 	}
 	r.packages = packages
@@ -273,17 +279,38 @@ func ParseRepomdXML(body io.ReadCloser) (Repomd, error) {
 	return result, err
 }
 
-// Unzips a gzipped body response, then parses the contained XML for package information
+// Unzips a compressed body response, then parses the contained XML for package information
 // Returns an array of package data
-func ParseCompressedXMLData(body io.ReadCloser) ([]Package, error) {
-	reader, err := gzip.NewReader(body)
+func ParseCompressedXMLData(body io.Reader) ([]Package, error) {
+	var reader, data io.Reader
+	var err error
+
+	data = body
+	readData, err := io.ReadAll(data)
+	if err != nil {
+		return []Package{}, err
+	}
+	fileType, err := filetype.Match(readData)
+	if err != nil {
+		return []Package{}, err
+	}
 
+	data = bytes.NewReader(readData)
+	switch fileType {
+	case matchers.TypeGz:
+		reader, err = gzip.NewReader(data)
+	case matchers.TypeZstd:
+		reader, err = zstd.NewReader(data)
+	case matchers.TypeXz:
+		reader, err = xz.NewReader(data)
+	default:
+		return []Package{}, fmt.Errorf("invalid file type: must be gzip, xz, or zstd.")
+	}
 	if err != nil {
 		return []Package{}, fmt.Errorf("Error unzipping response body: %w", err)
 	}
 
 	decoder := xml.NewDecoder(reader)
-	reader.Close()
 
 	result := []Package{}
 
diff --git a/pkg/yum/repository_test.go b/pkg/yum/repository_test.go
index 1e52c00..83205a4 100644
--- a/pkg/yum/repository_test.go
+++ b/pkg/yum/repository_test.go
@@ -187,28 +187,36 @@ func TestFetchRepomdSignature(t *testing.T) {
 	assert.Nil(t, err)
 }
 
-// Check that the parser can decompress a gzip file and read the correct number of packages
+// Check that the parser can decompress a compressed file and read the correct number of packages
 func TestParseCompressedXMLData(t *testing.T) {
-	xmlFile, err := os.Open("mocks/primary.xml.gz")
-	if err != nil {
-		log.Fatal(err)
+	paths := []string{
+		"mocks/primary.xml.gz",
+		"mocks/primary.xml.xz",
+		"mocks/primary.xml.zst",
 	}
-	defer xmlFile.Close()
-	result, err := ParseCompressedXMLData(xmlFile)
-	if err != nil {
-		t.Errorf("Error in test: %v", err)
-	}
-	if len(result) != 2 {
-		t.Errorf("Error - Expected to return 2 packages but received: %v", len(result))
-	}
-	if result[0].Checksum.Type != "sha1" {
-		t.Errorf(fmt.Sprintf("Checksum of %s received, should be sha1", result[0].Checksum.Type))
-	}
-	if result[0].Summary == "" {
-		t.Errorf("Did not properly parse summary")
-	}
-	if result[0].Checksum.Value == "" {
-		t.Errorf("Did not properly parse checksum")
+
+	for _, path := range paths {
+		xmlFile, err := os.Open(path)
+		if err != nil {
+			log.Fatal(err)
+		}
+		defer xmlFile.Close()
+		result, err := ParseCompressedXMLData(xmlFile)
+		if err != nil {
+			t.Errorf("Error in test: %v", err)
+		}
+		if len(result) != 2 {
+			t.Errorf("Error - Expected to return 2 packages but received: %v", len(result))
+		}
+		if result[0].Checksum.Type != "sha1" {
+			t.Errorf(fmt.Sprintf("Checksum of %s received, should be sha1", result[0].Checksum.Type))
+		}
+		if result[0].Summary == "" {
+			t.Errorf("Did not properly parse summary")
+		}
+		if result[0].Checksum.Value == "" {
+			t.Errorf("Did not properly parse checksum")
+		}
 	}
 }