Add chunked transfer encoding support for LFS uploads (#36380)

Enable chunked transfer encoding for Git LFS uploads by adding
Transfer-Encoding: chunked header to upload action responses. This
prevents large file uploads (100+ MB) from being blocked by reverse
proxies like Cloudflare that buffer non-chunked requests.

Fix https://github.com/go-gitea/gitea/issues/22233

---------

Co-authored-by: wxiaoguang <wxiaoguang@gmail.com>

modules/lfs/shared.go

@@ -66,6 +66,21 @@ type Link struct {
 	ExpiresAt *time.Time        `json:"expires_at,omitempty"`
 }
 
+func NewLink(href string) *Link {
+	return &Link{Href: href}
+}
+
+func (l *Link) WithHeader(k, v string) *Link {
+	if v == "" {
+		return l
+	}
+	if l.Header == nil {
+		l.Header = make(map[string]string)
+	}
+	l.Header[k] = v
+	return l
+}
+
 // ObjectError defines the JSON structure returned to the client in case of an error.
 type ObjectError struct {
 	Code    int    `json:"code"`

services/lfs/server.go

@@ -11,7 +11,6 @@ import (
 	"errors"
 	"fmt"
 	"io"
-	"maps"
 	"net/http"
 	"net/url"
 	"regexp"
@@ -487,40 +486,32 @@ func buildObjectResponse(rc *requestContext, pointer lfs_module.Pointer, downloa
 		rep.Error = err
 	} else {
 		rep.Actions = make(map[string]*lfs_module.Link)
-
-		header := make(map[string]string)
-
-		if len(rc.Authorization) > 0 {
-			header["Authorization"] = rc.Authorization
-		}
-
 		if download {
 			var link *lfs_module.Link
 			if setting.LFS.Storage.ServeDirect() {
 				// If we have a signed url (S3, object storage), redirect to this directly.
 				u, err := storage.LFS.URL(pointer.RelativePath(), pointer.Oid, rc.Method, nil)
 				if u != nil && err == nil {
-					// Presigned url does not need the Authorization header
-					// https://github.com/go-gitea/gitea/issues/21525
-					delete(header, "Authorization")
-					link = &lfs_module.Link{Href: u.String(), Header: header}
+					link = lfs_module.NewLink(u.String()) // Presigned url does not need the Authorization header
 				}
 			}
 			if link == nil {
-				link = &lfs_module.Link{Href: rc.DownloadLink(pointer), Header: header}
+				link = lfs_module.NewLink(rc.DownloadLink(pointer)).WithHeader("Authorization", rc.Authorization)
 			}
 			rep.Actions["download"] = link
 		}
 		if upload {
-			rep.Actions["upload"] = &lfs_module.Link{Href: rc.UploadLink(pointer), Header: header}
+			// Set Transfer-Encoding header to enable chunked uploads. Required by git-lfs client to do chunked transfer.
+			// See: https://github.com/git-lfs/git-lfs/blob/main/tq/basic_upload.go#L58-59
+			rep.Actions["upload"] = lfs_module.NewLink(rc.UploadLink(pointer)).
+				WithHeader("Authorization", rc.Authorization).
+				WithHeader("Transfer-Encoding", "chunked")
 
-			verifyHeader := make(map[string]string)
-			maps.Copy(verifyHeader, header)
-
-			// This is only needed to workaround https://github.com/git-lfs/git-lfs/issues/3662
-			verifyHeader["Accept"] = lfs_module.AcceptHeader
-
-			rep.Actions["verify"] = &lfs_module.Link{Href: rc.VerifyLink(pointer), Header: verifyHeader}
+			// "Accept" header is the workaround for git-lfs < 2.8.0 (before 2019).
+			// This workaround could be removed in the future: https://github.com/git-lfs/git-lfs/issues/3662
+			rep.Actions["verify"] = lfs_module.NewLink(rc.VerifyLink(pointer)).
+				WithHeader("Authorization", rc.Authorization).
+				WithHeader("Accept", lfs_module.AcceptHeader)
 		}
 	}
 	return rep

tests/integration/api_repo_lfs_test.go

@@ -317,6 +317,7 @@ func TestAPILFSBatch(t *testing.T) {
 			ul := br.Objects[0].Actions["upload"]
 			assert.NotNil(t, ul)
 			assert.NotEmpty(t, ul.Href)
+			assert.Equal(t, "chunked", ul.Header["Transfer-Encoding"], "git-lfs client needs Transfer-Encoding to do chunked transfer")
 			assert.Contains(t, br.Objects[0].Actions, "verify")
 			vl := br.Objects[0].Actions["verify"]
 			assert.NotNil(t, vl)