Add ability to download subpath archive (#36371)

closes: https://github.com/go-gitea/gitea/issues/4478

---------

Signed-off-by: wxiaoguang <wxiaoguang@gmail.com>
Co-authored-by: wxiaoguang <wxiaoguang@gmail.com>

.golangci.yml

@@ -48,6 +48,8 @@ linters:
               desc: do not use the ini package, use gitea's config system instead
             - pkg: gitea.com/go-chi/cache
               desc: do not use the go-chi cache package, use gitea's cache system
+            - pkg: github.com/pkg/errors
+              desc: use builtin errors package instead
     nolintlint:
       allow-unused: false
       require-explanation: true

go.mod

@@ -95,7 +95,6 @@ require (
 	github.com/olivere/elastic/v7 v7.0.32
 	github.com/opencontainers/go-digest v1.0.0
 	github.com/opencontainers/image-spec v1.1.1
-	github.com/pkg/errors v0.9.1
 	github.com/pquerna/otp v1.5.0
 	github.com/prometheus/client_golang v1.23.0
 	github.com/quasoft/websspi v1.1.2
@@ -251,6 +250,7 @@ require (
 	github.com/philhofer/fwd v1.2.0 // indirect
 	github.com/pierrec/lz4/v4 v4.1.22 // indirect
 	github.com/pjbgf/sha1cd v0.4.0 // indirect
+	github.com/pkg/errors v0.9.1 // indirect
 	github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect
 	github.com/prometheus/client_model v0.6.2 // indirect
 	github.com/prometheus/common v0.65.0 // indirect

modules/git/gitcmd/command.go

@@ -426,9 +426,9 @@ type RunStdError interface {
 }
 
 type runStdError struct {
-	err    error
-	stderr string
-	errMsg string
+	err    error  // usually the low-level error like `*exec.ExitError`
+	stderr string // git command's stderr output
+	errMsg string // the cached error message for Error() method
 }
 
 func (r *runStdError) Error() string {
@@ -448,6 +448,22 @@ func (r *runStdError) Stderr() string {
 	return r.stderr
 }
 
+func ErrorAsStderr(err error) (string, bool) {
+	var runErr RunStdError
+	if errors.As(err, &runErr) {
+		return runErr.Stderr(), true
+	}
+	return "", false
+}
+
+func StderrHasPrefix(err error, prefix string) bool {
+	stderr, ok := ErrorAsStderr(err)
+	if !ok {
+		return false
+	}
+	return strings.HasPrefix(stderr, prefix)
+}
+
 func IsErrorExitCode(err error, code int) bool {
 	var exitError *exec.ExitError
 	if errors.As(err, &exitError) {

modules/git/gitcmd/command_test.go

@@ -12,6 +12,7 @@ import (
 	"code.gitea.io/gitea/modules/tempdir"
 
 	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
 )
 
 func TestMain(m *testing.M) {
@@ -99,3 +100,14 @@ func TestCommandString(t *testing.T) {
 	cmd = NewCommand("url: https://a:b@c/", "/root/dir-a/dir-b")
 	assert.Equal(t, cmd.prog+` "url: https://sanitized-credential@c/" .../dir-a/dir-b`, cmd.LogString())
 }
+
+func TestRunStdError(t *testing.T) {
+	e := &runStdError{stderr: "some error"}
+	var err RunStdError = e
+
+	var asErr RunStdError
+	require.ErrorAs(t, err, &asErr)
+	require.Equal(t, "some error", asErr.Stderr())
+
+	require.ErrorAs(t, fmt.Errorf("wrapped %w", err), &asErr)
+}

modules/git/gitcmd/utils.go

@@ -3,12 +3,18 @@
 
 package gitcmd
 
-import "fmt"
+import (
+	"fmt"
 
-// ConcatenateError concatenats an error with stderr string
+	"code.gitea.io/gitea/modules/util"
+)
+
+// ConcatenateError concatenates an error with stderr string
+// FIXME: use RunStdError instead
 func ConcatenateError(err error, stderr string) error {
 	if len(stderr) == 0 {
 		return err
 	}
-	return fmt.Errorf("%w - %s", err, stderr)
+	errMsg := fmt.Sprintf("%s - %s", err.Error(), stderr)
+	return util.ErrorWrap(&runStdError{err: err, stderr: stderr, errMsg: errMsg}, "%s", errMsg)
 }

modules/gitrepo/archive.go

@@ -8,7 +8,9 @@ import (
 	"fmt"
 	"io"
 	"os"
+	"path"
 	"path/filepath"
+	"slices"
 	"strings"
 
 	"code.gitea.io/gitea/modules/git/gitcmd"
@@ -16,7 +18,7 @@ import (
 )
 
 // CreateArchive create archive content to the target path
-func CreateArchive(ctx context.Context, repo Repository, format string, target io.Writer, usePrefix bool, commitID string) error {
+func CreateArchive(ctx context.Context, repo Repository, format string, target io.Writer, usePrefix bool, commitID string, paths []string) error {
 	if format == "unknown" {
 		return fmt.Errorf("unknown format: %v", format)
 	}
@@ -28,6 +30,13 @@ func CreateArchive(ctx context.Context, repo Repository, format string, target i
 	cmd.AddOptionFormat("--format=%s", format)
 	cmd.AddDynamicArguments(commitID)
 
+	paths = slices.Clone(paths)
+	for i := range paths {
+		// although "git archive" already ensures the paths won't go outside the repo, we still clean them here for safety
+		paths[i] = path.Clean(paths[i])
+	}
+	cmd.AddDynamicArguments(paths...)
+
 	var stderr strings.Builder
 	if err := RunCmd(ctx, repo, cmd.WithStdout(target).WithStderr(&stderr)); err != nil {
 		return gitcmd.ConcatenateError(err, stderr.String())

modules/test/utils.go

@@ -4,6 +4,9 @@
 package test
 
 import (
+	"archive/tar"
+	"compress/gzip"
+	"io"
 	"net/http"
 	"net/http/httptest"
 	"os"
@@ -71,3 +74,31 @@ func SetupGiteaRoot() string {
 	_ = os.Setenv("GITEA_ROOT", giteaRoot)
 	return giteaRoot
 }
+
+func ReadAllTarGzContent(r io.Reader) (map[string]string, error) {
+	gzr, err := gzip.NewReader(r)
+	if err != nil {
+		return nil, err
+	}
+
+	content := make(map[string]string)
+
+	tr := tar.NewReader(gzr)
+	for {
+		hd, err := tr.Next()
+		if err == io.EOF {
+			break
+		}
+		if err != nil {
+			return nil, err
+		}
+
+		buf, err := io.ReadAll(tr)
+		if err != nil {
+			return nil, err
+		}
+
+		content[hd.Name] = string(buf)
+	}
+	return content, nil
+}

options/locale/locale_en-US.json

@@ -977,6 +977,7 @@
   "repo.fork.blocked_user": "Cannot fork the repository because you are blocked by the repository owner.",
   "repo.use_template": "Use this template",
   "repo.open_with_editor": "Open with %s",
+  "repo.download_directory_as": "Download directory as %s",
   "repo.download_zip": "Download ZIP",
   "repo.download_tar": "Download TAR.GZ",
   "repo.download_bundle": "Download BUNDLE",

routers/api/v1/repo/download.go

@@ -8,25 +8,35 @@ import (
 	"net/http"
 
 	repo_model "code.gitea.io/gitea/models/repo"
+	"code.gitea.io/gitea/modules/util"
 	"code.gitea.io/gitea/services/context"
 	archiver_service "code.gitea.io/gitea/services/repository/archiver"
 )
 
-func serveRepoArchive(ctx *context.APIContext, reqFileName string) {
-	aReq, err := archiver_service.NewRequest(ctx.Repo.Repository, ctx.Repo.GitRepo, reqFileName)
+func serveRepoArchive(ctx *context.APIContext, reqFileName string, paths []string) {
+	aReq, err := archiver_service.NewRequest(ctx.Repo.Repository, ctx.Repo.GitRepo, reqFileName, paths)
 	if err != nil {
-		if errors.Is(err, archiver_service.ErrUnknownArchiveFormat{}) {
+		if errors.Is(err, util.ErrInvalidArgument) {
 			ctx.APIError(http.StatusBadRequest, err)
-		} else if errors.Is(err, archiver_service.RepoRefNotFoundError{}) {
+		} else if errors.Is(err, util.ErrNotExist) {
 			ctx.APIError(http.StatusNotFound, err)
 		} else {
 			ctx.APIErrorInternal(err)
 		}
 		return
 	}
-	archiver_service.ServeRepoArchive(ctx.Base, aReq)
+	err = archiver_service.ServeRepoArchive(ctx.Base, aReq)
+	if err != nil {
+		if errors.Is(err, util.ErrInvalidArgument) {
+			ctx.APIError(http.StatusBadRequest, err)
+		} else {
+			ctx.APIErrorInternal(err)
+		}
+	}
 }
 
+// DownloadArchive is the GitHub-compatible endpoint to download repository archives
+// TODO: The API document is missing: Add github compatible tarball download API endpoints (#32572)
 func DownloadArchive(ctx *context.APIContext) {
 	var tp repo_model.ArchiveType
 	switch ballType := ctx.PathParam("ball_type"); ballType {
@@ -40,5 +50,5 @@ func DownloadArchive(ctx *context.APIContext) {
 		ctx.APIError(http.StatusBadRequest, "Unknown archive type: "+ballType)
 		return
 	}
-	serveRepoArchive(ctx, ctx.PathParam("*")+"."+tp.String())
+	serveRepoArchive(ctx, ctx.PathParam("*")+"."+tp.String(), ctx.FormStrings("path"))
 }

routers/api/v1/repo/file.go

@@ -273,13 +273,19 @@ func GetArchive(ctx *context.APIContext) {
 	//   description: the git reference for download with attached archive format (e.g. master.zip)
 	//   type: string
 	//   required: true
+	// - name: path
+	//   in: query
+	//   type: array
+	//   items:
+	//     type: string
+	//   description: subpath of the repository to download
+	//   collectionFormat: multi
 	// responses:
 	//   200:
 	//     description: success
 	//   "404":
 	//     "$ref": "#/responses/notFound"
-
-	serveRepoArchive(ctx, ctx.PathParam("*"))
+	serveRepoArchive(ctx, ctx.PathParam("*"), ctx.FormStrings("path"))
 }
 
 // GetEditorconfig get editor config of a repository

routers/web/repo/repo.go

@@ -364,31 +364,39 @@ func RedirectDownload(ctx *context.Context) {
 
 // Download an archive of a repository
 func Download(ctx *context.Context) {
-	aReq, err := archiver_service.NewRequest(ctx.Repo.Repository, ctx.Repo.GitRepo, ctx.PathParam("*"))
+	aReq, err := archiver_service.NewRequest(ctx.Repo.Repository, ctx.Repo.GitRepo, ctx.PathParam("*"), ctx.FormStrings("path"))
 	if err != nil {
-		if errors.Is(err, archiver_service.ErrUnknownArchiveFormat{}) {
+		if errors.Is(err, util.ErrInvalidArgument) {
 			ctx.HTTPError(http.StatusBadRequest, err.Error())
-		} else if errors.Is(err, archiver_service.RepoRefNotFoundError{}) {
+		} else if errors.Is(err, util.ErrNotExist) {
 			ctx.HTTPError(http.StatusNotFound, err.Error())
 		} else {
 			ctx.ServerError("archiver_service.NewRequest", err)
 		}
 		return
 	}
-	archiver_service.ServeRepoArchive(ctx.Base, aReq)
+	err = archiver_service.ServeRepoArchive(ctx.Base, aReq)
+	if err != nil {
+		if errors.Is(err, util.ErrInvalidArgument) {
+			ctx.HTTPError(http.StatusBadRequest, err.Error())
+		} else {
+			ctx.ServerError("archiver_service.ServeRepoArchive", err)
+		}
+	}
 }
 
 // InitiateDownload will enqueue an archival request, as needed.  It may submit
 // a request that's already in-progress, but the archiver service will just
 // kind of drop it on the floor if this is the case.
 func InitiateDownload(ctx *context.Context) {
-	if setting.Repository.StreamArchives {
+	paths := ctx.FormStrings("path")
+	if setting.Repository.StreamArchives || len(paths) > 0 {
 		ctx.JSON(http.StatusOK, map[string]any{
 			"complete": true,
 		})
 		return
 	}
-	aReq, err := archiver_service.NewRequest(ctx.Repo.Repository, ctx.Repo.GitRepo, ctx.PathParam("*"))
+	aReq, err := archiver_service.NewRequest(ctx.Repo.Repository, ctx.Repo.GitRepo, ctx.PathParam("*"), paths)
 	if err != nil {
 		ctx.HTTPError(http.StatusBadRequest, "invalid archive request")
 		return

services/pull/commit_status.go

@@ -6,6 +6,8 @@ package pull
 
 import (
 	"context"
+	"errors"
+	"fmt"
 
 	"code.gitea.io/gitea/models/db"
 	git_model "code.gitea.io/gitea/models/git"
@@ -14,8 +16,6 @@ import (
 	"code.gitea.io/gitea/modules/gitrepo"
 	"code.gitea.io/gitea/modules/glob"
 	"code.gitea.io/gitea/modules/log"
-
-	"github.com/pkg/errors"
 )
 
 // MergeRequiredContextsCommitStatus returns a commit status state for given required contexts
@@ -69,7 +69,7 @@ func MergeRequiredContextsCommitStatus(commitStatuses []*git_model.CommitStatus,
 func IsPullCommitStatusPass(ctx context.Context, pr *issues_model.PullRequest) (bool, error) {
 	pb, err := git_model.GetFirstMatchProtectedBranchRule(ctx, pr.BaseRepoID, pr.BaseBranch)
 	if err != nil {
-		return false, errors.Wrap(err, "GetLatestCommitStatus")
+		return false, fmt.Errorf("GetLatestCommitStatus: %w", err)
 	}
 	if pb == nil || !pb.EnableStatusCheck {
 		return true, nil
@@ -86,19 +86,19 @@ func IsPullCommitStatusPass(ctx context.Context, pr *issues_model.PullRequest) (
 func GetPullRequestCommitStatusState(ctx context.Context, pr *issues_model.PullRequest) (commitstatus.CommitStatusState, error) {
 	// Ensure HeadRepo is loaded
 	if err := pr.LoadHeadRepo(ctx); err != nil {
-		return "", errors.Wrap(err, "LoadHeadRepo")
+		return "", fmt.Errorf("LoadHeadRepo: %w", err)
 	}
 
 	// check if all required status checks are successful
 	headGitRepo, closer, err := gitrepo.RepositoryFromContextOrOpen(ctx, pr.HeadRepo)
 	if err != nil {
-		return "", errors.Wrap(err, "OpenRepository")
+		return "", fmt.Errorf("OpenRepository: %w", err)
 	}
 	defer closer.Close()
 
 	if pr.Flow == issues_model.PullRequestFlowGithub {
 		if exist, err := git_model.IsBranchExist(ctx, pr.HeadRepo.ID, pr.HeadBranch); err != nil {
-			return "", errors.Wrap(err, "IsBranchExist")
+			return "", fmt.Errorf("IsBranchExist: %w", err)
 		} else if !exist {
 			return "", errors.New("Head branch does not exist, can not merge")
 		}
@@ -118,17 +118,17 @@ func GetPullRequestCommitStatusState(ctx context.Context, pr *issues_model.PullR
 	}
 
 	if err := pr.LoadBaseRepo(ctx); err != nil {
-		return "", errors.Wrap(err, "LoadBaseRepo")
+		return "", fmt.Errorf("LoadBaseRepo: %w", err)
 	}
 
 	commitStatuses, err := git_model.GetLatestCommitStatus(ctx, pr.BaseRepo.ID, sha, db.ListOptionsAll)
 	if err != nil {
-		return "", errors.Wrap(err, "GetLatestCommitStatus")
+		return "", fmt.Errorf("GetLatestCommitStatus: %w", err)
 	}
 
 	pb, err := git_model.GetFirstMatchProtectedBranchRule(ctx, pr.BaseRepoID, pr.BaseBranch)
 	if err != nil {
-		return "", errors.Wrap(err, "LoadProtectedBranch")
+		return "", fmt.Errorf("LoadProtectedBranch: %w", err)
 	}
 	var requiredContexts []string
 	if pb != nil {

services/repository/archiver/archiver.go

@@ -8,7 +8,6 @@ import (
 	"errors"
 	"fmt"
 	"io"
-	"net/http"
 	"os"
 	"strings"
 	"time"
@@ -16,6 +15,7 @@ import (
 	"code.gitea.io/gitea/models/db"
 	repo_model "code.gitea.io/gitea/models/repo"
 	"code.gitea.io/gitea/modules/git"
+	"code.gitea.io/gitea/modules/git/gitcmd"
 	"code.gitea.io/gitea/modules/gitrepo"
 	"code.gitea.io/gitea/modules/graceful"
 	"code.gitea.io/gitea/modules/httplib"
@@ -24,6 +24,7 @@ import (
 	"code.gitea.io/gitea/modules/queue"
 	"code.gitea.io/gitea/modules/setting"
 	"code.gitea.io/gitea/modules/storage"
+	"code.gitea.io/gitea/modules/util"
 	gitea_context "code.gitea.io/gitea/services/context"
 )
 
@@ -36,58 +37,31 @@ type ArchiveRequest struct {
 	Repo     *repo_model.Repository
 	Type     repo_model.ArchiveType
 	CommitID string
+	Paths    []string
 
 	archiveRefShortName string // the ref short name to download the archive, for example: "master", "v1.0.0", "commit id"
 }
 
-// ErrUnknownArchiveFormat request archive format is not supported
-type ErrUnknownArchiveFormat struct {
-	RequestNameType string
-}
-
-// Error implements error
-func (err ErrUnknownArchiveFormat) Error() string {
-	return "unknown format: " + err.RequestNameType
-}
-
-// Is implements error
-func (ErrUnknownArchiveFormat) Is(err error) bool {
-	_, ok := err.(ErrUnknownArchiveFormat)
-	return ok
-}
-
-// RepoRefNotFoundError is returned when a requested reference (commit, tag) was not found.
-type RepoRefNotFoundError struct {
-	RefShortName string
-}
-
-// Error implements error.
-func (e RepoRefNotFoundError) Error() string {
-	return "unrecognized repository reference: " + e.RefShortName
-}
-
-func (e RepoRefNotFoundError) Is(err error) bool {
-	_, ok := err.(RepoRefNotFoundError)
-	return ok
-}
-
 // NewRequest creates an archival request, based on the URI.  The
 // resulting ArchiveRequest is suitable for being passed to Await()
 // if it's determined that the request still needs to be satisfied.
-func NewRequest(repo *repo_model.Repository, gitRepo *git.Repository, archiveRefExt string) (*ArchiveRequest, error) {
+func NewRequest(repo *repo_model.Repository, gitRepo *git.Repository, archiveRefExt string, paths []string) (*ArchiveRequest, error) {
 	// here the archiveRefShortName is not a clear ref, it could be a tag, branch or commit id
 	archiveRefShortName, archiveType := repo_model.SplitArchiveNameType(archiveRefExt)
 	if archiveType == repo_model.ArchiveUnknown {
-		return nil, ErrUnknownArchiveFormat{archiveRefExt}
+		return nil, util.NewInvalidArgumentErrorf("unknown format: %s", archiveRefExt)
+	}
+	if archiveType == repo_model.ArchiveBundle && len(paths) != 0 {
+		return nil, util.NewInvalidArgumentErrorf("cannot specify paths when requesting a bundle")
 	}
 
 	// Get corresponding commit.
 	commitID, err := gitRepo.ConvertToGitID(archiveRefShortName)
 	if err != nil {
-		return nil, RepoRefNotFoundError{RefShortName: archiveRefShortName}
+		return nil, util.NewNotExistErrorf("unrecognized repository reference: %s", archiveRefShortName)
 	}
 
-	r := &ArchiveRequest{Repo: repo, archiveRefShortName: archiveRefShortName, Type: archiveType}
+	r := &ArchiveRequest{Repo: repo, archiveRefShortName: archiveRefShortName, Type: archiveType, Paths: paths}
 	r.CommitID = commitID.String()
 	return r, nil
 }
@@ -159,6 +133,7 @@ func (aReq *ArchiveRequest) Stream(ctx context.Context, w io.Writer) error {
 		w,
 		setting.Repository.PrefixArchiveFiles,
 		aReq.CommitID,
+		aReq.Paths,
 	)
 }
 
@@ -339,7 +314,7 @@ func DeleteRepositoryArchives(ctx context.Context) error {
 	return storage.Clean(storage.RepoArchives)
 }
 
-func ServeRepoArchive(ctx *gitea_context.Base, archiveReq *ArchiveRequest) {
+func ServeRepoArchive(ctx *gitea_context.Base, archiveReq *ArchiveRequest) error {
 	// Add nix format link header so tarballs lock correctly:
 	// https://github.com/nixos/nix/blob/56763ff918eb308db23080e560ed2ea3e00c80a7/doc/manual/src/protocols/tarball-fetcher.md
 	ctx.Resp.Header().Add("Link", fmt.Sprintf(`<%s/archive/%s.%s?rev=%s>; rel="immutable"`,
@@ -350,20 +325,22 @@ func ServeRepoArchive(ctx *gitea_context.Base, archiveReq *ArchiveRequest) {
 	))
 	downloadName := archiveReq.Repo.Name + "-" + archiveReq.GetArchiveName()
 
-	if setting.Repository.StreamArchives {
+	if setting.Repository.StreamArchives || len(archiveReq.Paths) > 0 {
+		// the header must be set before starting streaming even an error would occur,
+		// because errors may happen in git command and such cases aren't in our control.
 		httplib.ServeSetHeaders(ctx.Resp, &httplib.ServeHeaderOptions{Filename: downloadName})
 		if err := archiveReq.Stream(ctx, ctx.Resp); err != nil && !ctx.Written() {
-			log.Error("Archive %v streaming failed: %v", archiveReq, err)
-			ctx.HTTPError(http.StatusInternalServerError)
+			if gitcmd.StderrHasPrefix(err, "fatal: pathspec") {
+				return util.NewInvalidArgumentErrorf("path doesn't exist or is invalid")
+			}
+			return fmt.Errorf("archive repo %s: failed to stream: %w", archiveReq.Repo.FullName(), err)
 		}
-		return
+		return nil
 	}
 
 	archiver, err := archiveReq.Await(ctx)
 	if err != nil {
-		log.Error("Archive %v await failed: %v", archiveReq, err)
-		ctx.HTTPError(http.StatusInternalServerError)
-		return
+		return fmt.Errorf("archive repo %s: failed to await: %w", archiveReq.Repo.FullName(), err)
 	}
 
 	rPath := archiver.RelativePath()
@@ -372,15 +349,13 @@ func ServeRepoArchive(ctx *gitea_context.Base, archiveReq *ArchiveRequest) {
 		u, err := storage.RepoArchives.URL(rPath, downloadName, ctx.Req.Method, nil)
 		if u != nil && err == nil {
 			ctx.Redirect(u.String())
-			return
+			return nil
 		}
 	}
 
 	fr, err := storage.RepoArchives.Open(rPath)
 	if err != nil {
-		log.Error("Archive %v open file failed: %v", archiveReq, err)
-		ctx.HTTPError(http.StatusInternalServerError)
-		return
+		return fmt.Errorf("archive repo %s: failed to open archive file: %w", archiveReq.Repo.FullName(), err)
 	}
 	defer fr.Close()
 
@@ -388,4 +363,5 @@ func ServeRepoArchive(ctx *gitea_context.Base, archiveReq *ArchiveRequest) {
 		Filename:     downloadName,
 		LastModified: archiver.CreatedUnix.AsLocalTime(),
 	})
+	return nil
 }

services/repository/archiver/archiver_test.go

@@ -8,11 +8,13 @@ import (
 	"time"
 
 	"code.gitea.io/gitea/models/unittest"
+	"code.gitea.io/gitea/modules/util"
 	"code.gitea.io/gitea/services/contexttest"
 
 	_ "code.gitea.io/gitea/models/actions"
 
 	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
 )
 
 func TestMain(m *testing.M) {
@@ -29,47 +31,47 @@ func TestArchive_Basic(t *testing.T) {
 	contexttest.LoadGitRepo(t, ctx)
 	defer ctx.Repo.GitRepo.Close()
 
-	bogusReq, err := NewRequest(ctx.Repo.Repository, ctx.Repo.GitRepo, firstCommit+".zip")
+	bogusReq, err := NewRequest(ctx.Repo.Repository, ctx.Repo.GitRepo, firstCommit+".zip", nil)
 	assert.NoError(t, err)
 	assert.NotNil(t, bogusReq)
 	assert.Equal(t, firstCommit+".zip", bogusReq.GetArchiveName())
 
 	// Check a series of bogus requests.
 	// Step 1, valid commit with a bad extension.
-	bogusReq, err = NewRequest(ctx.Repo.Repository, ctx.Repo.GitRepo, firstCommit+".unknown")
+	bogusReq, err = NewRequest(ctx.Repo.Repository, ctx.Repo.GitRepo, firstCommit+".unknown", nil)
 	assert.Error(t, err)
 	assert.Nil(t, bogusReq)
 
 	// Step 2, missing commit.
-	bogusReq, err = NewRequest(ctx.Repo.Repository, ctx.Repo.GitRepo, "dbffff.zip")
+	bogusReq, err = NewRequest(ctx.Repo.Repository, ctx.Repo.GitRepo, "dbffff.zip", nil)
 	assert.Error(t, err)
 	assert.Nil(t, bogusReq)
 
 	// Step 3, doesn't look like branch/tag/commit.
-	bogusReq, err = NewRequest(ctx.Repo.Repository, ctx.Repo.GitRepo, "db.zip")
+	bogusReq, err = NewRequest(ctx.Repo.Repository, ctx.Repo.GitRepo, "db.zip", nil)
 	assert.Error(t, err)
 	assert.Nil(t, bogusReq)
 
-	bogusReq, err = NewRequest(ctx.Repo.Repository, ctx.Repo.GitRepo, "master.zip")
+	bogusReq, err = NewRequest(ctx.Repo.Repository, ctx.Repo.GitRepo, "master.zip", nil)
 	assert.NoError(t, err)
 	assert.NotNil(t, bogusReq)
 	assert.Equal(t, "master.zip", bogusReq.GetArchiveName())
 
-	bogusReq, err = NewRequest(ctx.Repo.Repository, ctx.Repo.GitRepo, "test/archive.zip")
+	bogusReq, err = NewRequest(ctx.Repo.Repository, ctx.Repo.GitRepo, "test/archive.zip", nil)
 	assert.NoError(t, err)
 	assert.NotNil(t, bogusReq)
 	assert.Equal(t, "test-archive.zip", bogusReq.GetArchiveName())
 
 	// Now two valid requests, firstCommit with valid extensions.
-	zipReq, err := NewRequest(ctx.Repo.Repository, ctx.Repo.GitRepo, firstCommit+".zip")
+	zipReq, err := NewRequest(ctx.Repo.Repository, ctx.Repo.GitRepo, firstCommit+".zip", nil)
 	assert.NoError(t, err)
 	assert.NotNil(t, zipReq)
 
-	tgzReq, err := NewRequest(ctx.Repo.Repository, ctx.Repo.GitRepo, firstCommit+".tar.gz")
+	tgzReq, err := NewRequest(ctx.Repo.Repository, ctx.Repo.GitRepo, firstCommit+".tar.gz", nil)
 	assert.NoError(t, err)
 	assert.NotNil(t, tgzReq)
 
-	secondReq, err := NewRequest(ctx.Repo.Repository, ctx.Repo.GitRepo, secondCommit+".bundle")
+	secondReq, err := NewRequest(ctx.Repo.Repository, ctx.Repo.GitRepo, secondCommit+".bundle", nil)
 	assert.NoError(t, err)
 	assert.NotNil(t, secondReq)
 
@@ -89,7 +91,7 @@ func TestArchive_Basic(t *testing.T) {
 	// Sleep two seconds to make sure the queue doesn't change.
 	time.Sleep(2 * time.Second)
 
-	zipReq2, err := NewRequest(ctx.Repo.Repository, ctx.Repo.GitRepo, firstCommit+".zip")
+	zipReq2, err := NewRequest(ctx.Repo.Repository, ctx.Repo.GitRepo, firstCommit+".zip", nil)
 	assert.NoError(t, err)
 	// This zipReq should match what's sitting in the queue, as we haven't
 	// let it release yet.  From the consumer's point of view, this looks like
@@ -104,12 +106,12 @@ func TestArchive_Basic(t *testing.T) {
 	// Now we'll submit a request and TimedWaitForCompletion twice, before and
 	// after we release it.  We should trigger both the timeout and non-timeout
 	// cases.
-	timedReq, err := NewRequest(ctx.Repo.Repository, ctx.Repo.GitRepo, secondCommit+".tar.gz")
+	timedReq, err := NewRequest(ctx.Repo.Repository, ctx.Repo.GitRepo, secondCommit+".tar.gz", nil)
 	assert.NoError(t, err)
 	assert.NotNil(t, timedReq)
 	doArchive(t.Context(), timedReq)
 
-	zipReq2, err = NewRequest(ctx.Repo.Repository, ctx.Repo.GitRepo, firstCommit+".zip")
+	zipReq2, err = NewRequest(ctx.Repo.Repository, ctx.Repo.GitRepo, firstCommit+".zip", nil)
 	assert.NoError(t, err)
 	// Now, we're guaranteed to have released the original zipReq from the queue.
 	// Ensure that we don't get handed back the released entry somehow, but they
@@ -124,9 +126,13 @@ func TestArchive_Basic(t *testing.T) {
 	// Ideally, the extension would match what we originally requested.
 	assert.NotEqual(t, zipReq.GetArchiveName(), tgzReq.GetArchiveName())
 	assert.NotEqual(t, zipReq.GetArchiveName(), secondReq.GetArchiveName())
-}
 
-func TestErrUnknownArchiveFormat(t *testing.T) {
-	err := ErrUnknownArchiveFormat{RequestNameType: "xxx"}
-	assert.ErrorIs(t, err, ErrUnknownArchiveFormat{})
+	t.Run("BadPath", func(t *testing.T) {
+		badRequest, err := NewRequest(ctx.Repo.Repository, ctx.Repo.GitRepo, firstCommit+".tar.gz", []string{"not-a-path"})
+		require.NoError(t, err)
+		err = ServeRepoArchive(ctx.Base, badRequest)
+		require.Error(t, err)
+		assert.ErrorIs(t, err, util.ErrInvalidArgument)
+		assert.ErrorContains(t, err, "path doesn't exist or is invalid")
+	})
 }

templates/repo/view_content.tmpl

@@ -100,6 +100,11 @@
 					<a class="item" data-clipboard-text="{{.Repository.Link}}/src/commit/{{.CommitID}}/{{PathEscapeSegments .TreePath}}" data-clipboard-text-type="url">
 						{{svg "octicon-link" 16}}{{ctx.Locale.Tr "repo.file_copy_permalink"}}
 					</a>
+					{{if and (not $.DisableDownloadSourceArchives) $.RefFullName}}
+						<div class="divider"></div>
+						<a class="item muted archive-link" href="{{$.RepoLink}}/archive/{{PathEscapeSegments $.RefFullName.ShortName}}.zip?path={{PathEscapeSegments .TreePath}}" rel="nofollow">{{svg "octicon-file-zip"}}{{ctx.Locale.Tr "repo.download_directory_as" "ZIP"}}</a>
+						<a class="item muted archive-link" href="{{$.RepoLink}}/archive/{{PathEscapeSegments $.RefFullName.ShortName}}.tar.gz?path={{PathEscapeSegments .TreePath}}" rel="nofollow">{{svg "octicon-file-zip"}}{{ctx.Locale.Tr "repo.download_directory_as" "TAR.GZ"}}</a>
+					{{end}}
 					{{if and (.Permission.CanWrite ctx.Consts.RepoUnitTypeCode) (not .Repository.IsArchived) (not $isTreePathRoot)}}
 						<div class="divider"></div>
 						<a class="item tw-text-danger" href="{{.RepoLink}}/_delete/{{.BranchName | PathEscapeSegments}}/{{.TreePath | PathEscapeSegments}}">

templates/swagger/v1_json.tmpl

@@ -6256,6 +6256,16 @@
             "name": "archive",
             "in": "path",
             "required": true
+          },
+          {
+            "type": "array",
+            "items": {
+              "type": "string"
+            },
+            "collectionFormat": "multi",
+            "description": "subpath of the repository to download",
+            "name": "path",
+            "in": "query"
           }
         ],
         "responses": {

tests/integration/api_packages_arch_test.go

@@ -16,6 +16,7 @@ import (
 	"code.gitea.io/gitea/models/unittest"
 	user_model "code.gitea.io/gitea/models/user"
 	arch_module "code.gitea.io/gitea/modules/packages/arch"
+	"code.gitea.io/gitea/modules/test"
 	arch_service "code.gitea.io/gitea/services/packages/arch"
 	"code.gitea.io/gitea/tests"
 
@@ -78,34 +79,6 @@ license = MIT`)
 
 		return buf.Bytes()
 	}
-	readIndexContent := func(r io.Reader) (map[string]string, error) {
-		gzr, err := gzip.NewReader(r)
-		if err != nil {
-			return nil, err
-		}
-
-		content := make(map[string]string)
-
-		tr := tar.NewReader(gzr)
-		for {
-			hd, err := tr.Next()
-			if err == io.EOF {
-				break
-			}
-			if err != nil {
-				return nil, err
-			}
-
-			buf, err := io.ReadAll(tr)
-			if err != nil {
-				return nil, err
-			}
-
-			content[hd.Name] = string(buf)
-		}
-
-		return content, nil
-	}
 
 	compressions := []string{"gz", "xz", "zst"}
 	repositories := []string{"main", "testing", "with/slash", ""}
@@ -204,7 +177,7 @@ license = MIT`)
 					req := NewRequest(t, "GET", fmt.Sprintf("%s/%s/aarch64/%s", rootURL, repository, arch_service.IndexArchiveFilename))
 					resp := MakeRequest(t, req, http.StatusOK)
 
-					content, err := readIndexContent(resp.Body)
+					content, err := test.ReadAllTarGzContent(resp.Body)
 					assert.NoError(t, err)
 
 					desc, has := content[fmt.Sprintf("%s-%s/desc", packageName, packageVersion)]
@@ -256,7 +229,7 @@ license = MIT`)
 					req = NewRequest(t, "GET", fmt.Sprintf("%s/%s/aarch64/%s", rootURL, repository, arch_service.IndexArchiveFilename))
 					resp := MakeRequest(t, req, http.StatusOK)
 
-					content, err := readIndexContent(resp.Body)
+					content, err := test.ReadAllTarGzContent(resp.Body)
 					assert.NoError(t, err)
 
 					desc, has := content[fmt.Sprintf("%s-%s/desc", packageName, packageVersion)]
@@ -311,7 +284,7 @@ license = MIT`)
 		req = NewRequest(t, "GET", fmt.Sprintf("%s/aarch64/%s", rootURL, arch_service.IndexArchiveFilename))
 		resp := MakeRequest(t, req, http.StatusOK)
 
-		content, err := readIndexContent(resp.Body)
+		content, err := test.ReadAllTarGzContent(resp.Body)
 		assert.NoError(t, err)
 		assert.Len(t, content, 2)
 
@@ -326,7 +299,7 @@ license = MIT`)
 
 		req = NewRequest(t, "GET", fmt.Sprintf("%s/aarch64/%s", rootURL, arch_service.IndexArchiveFilename))
 		resp = MakeRequest(t, req, http.StatusOK)
-		content, err = readIndexContent(resp.Body)
+		content, err = test.ReadAllTarGzContent(resp.Body)
 		assert.NoError(t, err)
 		assert.Len(t, content, 2)
 		_, has = content["gitea-test-1.0.0/desc"]

tests/integration/repo_archive_test.go

@@ -15,6 +15,7 @@ import (
 	"code.gitea.io/gitea/tests"
 
 	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
 )
 
 func TestRepoDownloadArchive(t *testing.T) {
@@ -23,11 +24,35 @@ func TestRepoDownloadArchive(t *testing.T) {
 	defer test.MockVariableValue(&web.GzipMinSize, 10)()
 	defer test.MockVariableValue(&testWebRoutes, routers.NormalRoutes())()
 
-	req := NewRequest(t, "GET", "/user2/repo1/archive/master.zip")
-	req.Header.Set("Accept-Encoding", "gzip")
-	resp := MakeRequest(t, req, http.StatusOK)
-	bs, err := io.ReadAll(resp.Body)
-	assert.NoError(t, err)
-	assert.Empty(t, resp.Header().Get("Content-Encoding"))
-	assert.Len(t, bs, 320)
+	t.Run("NoDuplicateCompression", func(t *testing.T) {
+		req := NewRequest(t, "GET", "/user2/repo1/archive/master.zip")
+		req.Header.Set("Accept-Encoding", "gzip")
+		resp := MakeRequest(t, req, http.StatusOK)
+		bs, err := io.ReadAll(resp.Body)
+		assert.NoError(t, err)
+		assert.Empty(t, resp.Header().Get("Content-Encoding"))
+		assert.Len(t, bs, 320)
+	})
+
+	t.Run("SubPath", func(t *testing.T) {
+		// When using "archiving and caching" approach, archiving with paths will always use streaming and never be cached
+		defer test.MockVariableValue(&setting.Repository.StreamArchives, false) // this can be removed if there is always streaming mode
+		req := NewRequest(t, "GET", "/user2/glob/archive/master.tar.gz?path=aaa.doc&path=x/y")
+		resp := MakeRequest(t, req, http.StatusOK)
+		content, err := test.ReadAllTarGzContent(resp.Body)
+		require.NoError(t, err)
+		assert.Empty(t, content["glob/a.txt"])
+		assert.NotEmpty(t, content["glob/aaa.doc"])
+		assert.Empty(t, content["glob/x/b.txt"])
+		assert.NotEmpty(t, content["glob/x/y/a.txt"])
+
+		req = NewRequest(t, "GET", "/user2/glob/archive/master.tar.gz")
+		resp = MakeRequest(t, req, http.StatusOK)
+		content, err = test.ReadAllTarGzContent(resp.Body)
+		require.NoError(t, err)
+		assert.NotEmpty(t, content["glob/a.txt"])
+		assert.NotEmpty(t, content["glob/aaa.doc"])
+		assert.NotEmpty(t, content["glob/x/b.txt"])
+		assert.NotEmpty(t, content["glob/x/y/a.txt"])
+	})
 }