Don't rely on transparent decompression in sync list (#2710)

* Don't rely on transparent decompression in sync list

The current sync list code relies on the HTTP client library transparently
decompressing the packages.gz file. This only works if the server provides
the 'Content-Encoding: gzip" HTTP header, which has apparently changed
recently.

This patch explicitly decompresses the package list if no encoding
information is provided by the server.

Fixes: https://github.com/Jguer/yay/issues/2709

v2: added gzip closer and added error handling

v3: blindly attempt to decompress, since the content encoding is
stripped by the HTTP client when transparently decompressing.

v4: also fix the fetch in command completion using common interface

Signed-off-by: Edwin Peer <espeer@gmail.com>

* fix failing tests, add fallback logging, add gz testcase

* fix lint

---------

Signed-off-by: Edwin Peer <espeer@gmail.com>

pkg/completion/completion.go

@@ -1,34 +1,27 @@
 package completion
 
 import (
-	"bufio"
 	"context"
-	"fmt"
 	"io"
-	"net/http"
-	"net/url"
 	"os"
-	"path"
 	"path/filepath"
 	"strings"
 	"time"
 
 	"github.com/Jguer/yay/v12/pkg/db"
+	"github.com/Jguer/yay/v12/pkg/download"
+	"github.com/Jguer/yay/v12/pkg/text"
 )
 
 type PkgSynchronizer interface {
 	SyncPackages(...string) []db.IPackage
 }
 
-type httpRequestDoer interface {
-	Do(req *http.Request) (*http.Response, error)
-}
-
 // Show provides completion info for shells.
-func Show(ctx context.Context, httpClient httpRequestDoer,
-	dbExecutor PkgSynchronizer, aurURL, completionPath string, interval int, force bool,
+func Show(ctx context.Context, httpClient download.HTTPRequestDoer,
+	dbExecutor PkgSynchronizer, aurURL, completionPath string, interval int, force bool, logger *text.Logger,
 ) error {
-	err := Update(ctx, httpClient, dbExecutor, aurURL, completionPath, interval, force)
+	err := Update(ctx, httpClient, dbExecutor, aurURL, completionPath, interval, force, logger)
 	if err != nil {
 		return err
 	}
@@ -45,8 +38,8 @@ func Show(ctx context.Context, httpClient httpRequestDoer,
 }
 
 // Update updates completion cache to be used by Complete.
-func Update(ctx context.Context, httpClient httpRequestDoer,
-	dbExecutor PkgSynchronizer, aurURL, completionPath string, interval int, force bool,
+func Update(ctx context.Context, httpClient download.HTTPRequestDoer,
+	dbExecutor PkgSynchronizer, aurURL, completionPath string, interval int, force bool, logger *text.Logger,
 ) error {
 	info, err := os.Stat(completionPath)
 
@@ -61,7 +54,7 @@ func Update(ctx context.Context, httpClient httpRequestDoer,
 			return errf
 		}
 
-		if createAURList(ctx, httpClient, aurURL, out) != nil {
+		if createAURList(ctx, httpClient, aurURL, out, logger) != nil {
 			defer os.Remove(completionPath)
 		}
 
@@ -75,41 +68,23 @@ func Update(ctx context.Context, httpClient httpRequestDoer,
 	return nil
 }
 
-// CreateAURList creates a new completion file.
-func createAURList(ctx context.Context, client httpRequestDoer, aurURL string, out io.Writer) error {
-	u, err := url.Parse(aurURL)
+// createAURList creates a new completion file.
+func createAURList(ctx context.Context, client download.HTTPRequestDoer, aurURL string, out io.Writer, logger *text.Logger) error {
+	scanner, err := download.GetPackageScanner(ctx, client, aurURL, logger)
 	if err != nil {
 		return err
 	}
-
-	u.Path = path.Join(u.Path, "packages.gz")
-
-	req, err := http.NewRequestWithContext(ctx, http.MethodGet, u.String(), http.NoBody)
-	if err != nil {
-		return err
-	}
-
-	resp, err := client.Do(req)
-	if err != nil {
-		return err
-	}
-	defer resp.Body.Close()
-
-	if resp.StatusCode != http.StatusOK {
-		return fmt.Errorf("invalid status code: %d", resp.StatusCode)
-	}
-
-	scanner := bufio.NewScanner(resp.Body)
+	defer scanner.Close()
 
 	scanner.Scan()
 
 	for scanner.Scan() {
-		text := scanner.Text()
-		if strings.HasPrefix(text, "#") {
+		pkgName := scanner.Text()
+		if strings.HasPrefix(pkgName, "#") {
 			continue
 		}
 
-		if _, err := io.WriteString(out, text+"\tAUR\n"); err != nil {
+		if _, err := io.WriteString(out, pkgName+"\tAUR\n"); err != nil {
 			return err
 		}
 	}

pkg/completion/completion_test.go

@@ -5,6 +5,7 @@ package completion
 
 import (
 	"bytes"
+	"compress/gzip"
 	"context"
 	"errors"
 	"io"
@@ -38,31 +39,55 @@ eternallands-sound	AUR
 
 type mockDoer struct {
 	t                *testing.T
-	returnBody       string
+	returnBody       []byte
 	returnStatusCode int
 	returnErr        error
-	wantUrl          string
+	wantURL          string
 }
 
-func (m *mockDoer) Do(req *http.Request) (*http.Response, error) {
-	assert.Equal(m.t, m.wantUrl, req.URL.String())
+func (m *mockDoer) Get(url string) (*http.Response, error) {
+	assert.Equal(m.t, m.wantURL, url)
 	return &http.Response{
 		StatusCode: m.returnStatusCode,
-		Body:       io.NopCloser(bytes.NewBufferString(m.returnBody)),
+		Body:       io.NopCloser(bytes.NewReader(m.returnBody)),
 	}, m.returnErr
 }
 
+func gzipString(s string) []byte {
+	var buf bytes.Buffer
+	gz := gzip.NewWriter(&buf)
+	gz.Write([]byte(s))
+	gz.Close()
+	return buf.Bytes()
+}
+
 func Test_createAURList(t *testing.T) {
 	t.Parallel()
 	doer := &mockDoer{
 		t:                t,
-		wantUrl:          "https://aur.archlinux.org/packages.gz",
+		wantURL:          "https://aur.archlinux.org/packages.gz",
 		returnStatusCode: 200,
-		returnBody:       samplePackageResp,
+		returnBody:       []byte(samplePackageResp),
 		returnErr:        nil,
 	}
 	out := &bytes.Buffer{}
-	err := createAURList(context.Background(), doer, "https://aur.archlinux.org", out)
+	err := createAURList(context.Background(), doer, "https://aur.archlinux.org", out, nil)
+	assert.NoError(t, err)
+	gotOut := out.String()
+	assert.Equal(t, expectPackageCompletion, gotOut)
+}
+
+func Test_createAURListGzip(t *testing.T) {
+	t.Parallel()
+	doer := &mockDoer{
+		t:                t,
+		wantURL:          "https://aur.archlinux.org/packages.gz",
+		returnStatusCode: 200,
+		returnBody:       gzipString(samplePackageResp),
+		returnErr:        nil,
+	}
+	out := &bytes.Buffer{}
+	err := createAURList(context.Background(), doer, "https://aur.archlinux.org", out, nil)
 	assert.NoError(t, err)
 	gotOut := out.String()
 	assert.Equal(t, expectPackageCompletion, gotOut)
@@ -72,14 +97,14 @@ func Test_createAURListHTTPError(t *testing.T) {
 	t.Parallel()
 	doer := &mockDoer{
 		t:                t,
-		wantUrl:          "https://aur.archlinux.org/packages.gz",
+		wantURL:          "https://aur.archlinux.org/packages.gz",
 		returnStatusCode: 200,
-		returnBody:       samplePackageResp,
+		returnBody:       []byte(samplePackageResp),
 		returnErr:        errors.New("Not available"),
 	}
 
 	out := &bytes.Buffer{}
-	err := createAURList(context.Background(), doer, "https://aur.archlinux.org", out)
+	err := createAURList(context.Background(), doer, "https://aur.archlinux.org", out, nil)
 	assert.EqualError(t, err, "Not available")
 }
 
@@ -87,13 +112,13 @@ func Test_createAURListStatusError(t *testing.T) {
 	t.Parallel()
 	doer := &mockDoer{
 		t:                t,
-		wantUrl:          "https://aur.archlinux.org/packages.gz",
+		wantURL:          "https://aur.archlinux.org/packages.gz",
 		returnStatusCode: 503,
-		returnBody:       samplePackageResp,
+		returnBody:       []byte(samplePackageResp),
 		returnErr:        nil,
 	}
 
 	out := &bytes.Buffer{}
-	err := createAURList(context.Background(), doer, "https://aur.archlinux.org", out)
+	err := createAURList(context.Background(), doer, "https://aur.archlinux.org", out, nil)
 	assert.EqualError(t, err, "invalid status code: 503")
 }

pkg/download/abs.go

@@ -59,7 +59,7 @@ func convertPkgNameForURL(pkgName string) string {
 }
 
 // ABSPKGBUILD retrieves the PKGBUILD file to a dest directory.
-func ABSPKGBUILD(httpClient httpRequestDoer, dbName, pkgName string) ([]byte, error) {
+func ABSPKGBUILD(httpClient HTTPRequestDoer, dbName, pkgName string) ([]byte, error) {
 	packageURL := getPackagePKGBUILDURL(pkgName)
 
 	resp, err := httpClient.Get(packageURL)

pkg/download/aur.go

@@ -1,11 +1,15 @@
 package download
 
 import (
+	"bufio"
+	"bytes"
+	"compress/gzip"
 	"context"
 	"fmt"
 	"io"
 	"net/http"
 	"net/url"
+	"path"
 	"sync"
 
 	"github.com/leonelquinteros/gotext"
@@ -15,7 +19,7 @@ import (
 	"github.com/Jguer/yay/v12/pkg/text"
 )
 
-func AURPKGBUILD(httpClient httpRequestDoer, pkgName, aurURL string) ([]byte, error) {
+func AURPKGBUILD(httpClient HTTPRequestDoer, pkgName, aurURL string) ([]byte, error) {
 	values := url.Values{}
 	values.Set("h", pkgName)
 	pkgURL := aurURL + "/cgit/aur.git/plain/PKGBUILD?" + values.Encode()
@@ -98,3 +102,69 @@ func AURPKGBUILDRepos(
 
 	return cloned, errs.Return()
 }
+
+// ScannerCloser combines a bufio.Scanner with a Close method.
+type ScannerCloser struct {
+	*bufio.Scanner
+	closer io.Closer
+}
+
+// Close closes the underlying gzip reader if present.
+func (s *ScannerCloser) Close() error {
+	if s.closer != nil {
+		return s.closer.Close()
+	}
+	return nil
+}
+
+// GetPackageScanner fetches the AUR packages.gz file and returns a scanner for reading its contents.
+// The caller must call Close() on the returned ScannerCloser when done to properly release resources.
+func GetPackageScanner(ctx context.Context, client HTTPRequestDoer, aurURL string, logger *text.Logger) (*ScannerCloser, error) {
+	u, err := url.Parse(aurURL)
+	if err != nil {
+		return nil, err
+	}
+
+	u.Path = path.Join(u.Path, "packages.gz")
+	packagesURL := u.String()
+
+	resp, err := client.Get(packagesURL)
+	if err != nil {
+		return nil, err
+	}
+
+	if resp.StatusCode != http.StatusOK {
+		resp.Body.Close()
+		return nil, fmt.Errorf("invalid status code: %d", resp.StatusCode)
+	}
+
+	// Read the entire body to allow trying gzip decompression
+	body, err := io.ReadAll(resp.Body)
+	resp.Body.Close()
+
+	if err != nil {
+		return nil, err
+	}
+
+	// Try to decompress as gzip; if that fails, use raw body
+	var reader io.Reader
+	var closer io.Closer
+
+	gzReader, gzErr := gzip.NewReader(bytes.NewReader(body))
+	if gzErr == nil {
+		reader = gzReader
+		closer = gzReader
+	} else {
+		if logger != nil {
+			logger.Debugln("gzip decompression not needed, using raw response body")
+		}
+		reader = bytes.NewReader(body)
+	}
+
+	scanner := bufio.NewScanner(reader)
+
+	return &ScannerCloser{
+		Scanner: scanner,
+		closer:  closer,
+	}, nil
+}

pkg/download/unified.go

@@ -18,7 +18,8 @@ import (
 	"github.com/Jguer/yay/v12/pkg/text"
 )
 
-type httpRequestDoer interface {
+// HTTPRequestDoer is an interface for HTTP clients that can perform GET requests.
+type HTTPRequestDoer interface {
 	Get(string) (*http.Response, error)
 }
 

pkg/sync/sync.go

@@ -65,7 +65,7 @@ func (o *OperationService) Run(ctx context.Context, run *runtime.Runtime,
 
 	go func() {
 		errComp := completion.Update(ctx, run.HTTPClient, o.dbExecutor,
-			o.cfg.AURURL, o.cfg.CompletionPath, o.cfg.CompletionInterval, false)
+			o.cfg.AURURL, o.cfg.CompletionPath, o.cfg.CompletionInterval, false, o.logger)
 		if errComp != nil {
 			o.logger.Warnln(errComp)
 		}