From 0ad94dfc70a7d45169989cac499f809b7df5a25e Mon Sep 17 00:00:00 2001 From: Lunny Xiao Date: Mon, 29 Dec 2025 10:19:42 -0800 Subject: [PATCH] Move catfile batch to a sub package of git module (#36232) --- modules/git/batch.go | 47 --- modules/git/batch_reader.go | 323 ++---------------- modules/git/blob_nogogit.go | 11 +- modules/git/catfile/batch.go | 178 ++++++++++ modules/git/catfile/reader.go | 211 ++++++++++++ .../languagestats/language_stats_nogogit.go | 5 +- modules/git/pipeline/lfs_nogogit.go | 5 +- modules/git/repo_base_nogogit.go | 34 +- modules/git/repo_branch_nogogit.go | 12 +- modules/git/repo_commit_nogogit.go | 29 +- modules/git/repo_tag_nogogit.go | 11 +- modules/git/repo_tree_nogogit.go | 6 +- modules/git/tree_entry_nogogit.go | 6 +- modules/git/tree_nogogit.go | 4 +- modules/gitrepo/cat_file.go | 6 +- modules/indexer/code/bleve/bleve.go | 15 +- .../code/elasticsearch/elasticsearch.go | 9 +- 17 files changed, 500 insertions(+), 412 deletions(-) delete mode 100644 modules/git/batch.go create mode 100644 modules/git/catfile/batch.go create mode 100644 modules/git/catfile/reader.go diff --git a/modules/git/batch.go b/modules/git/batch.go deleted file mode 100644 index f9e1748b54..0000000000 --- a/modules/git/batch.go +++ /dev/null @@ -1,47 +0,0 @@ -// Copyright 2024 The Gitea Authors. All rights reserved. -// SPDX-License-Identifier: MIT - -package git - -import ( - "bufio" - "context" -) - -type Batch struct { - cancel context.CancelFunc - Reader *bufio.Reader - Writer WriteCloserError -} - -// NewBatch creates a new batch for the given repository, the Close must be invoked before release the batch -func NewBatch(ctx context.Context, repoPath string) (*Batch, error) { - // Now because of some insanity with git cat-file not immediately failing if not run in a valid git directory we need to run git rev-parse first! - if err := ensureValidGitRepository(ctx, repoPath); err != nil { - return nil, err - } - - var batch Batch - batch.Writer, batch.Reader, batch.cancel = catFileBatch(ctx, repoPath) - return &batch, nil -} - -func NewBatchCheck(ctx context.Context, repoPath string) (*Batch, error) { - // Now because of some insanity with git cat-file not immediately failing if not run in a valid git directory we need to run git rev-parse first! - if err := ensureValidGitRepository(ctx, repoPath); err != nil { - return nil, err - } - - var check Batch - check.Writer, check.Reader, check.cancel = catFileBatchCheck(ctx, repoPath) - return &check, nil -} - -func (b *Batch) Close() { - if b.cancel != nil { - b.cancel() - b.Reader = nil - b.Writer = nil - b.cancel = nil - } -} diff --git a/modules/git/batch_reader.go b/modules/git/batch_reader.go index b5cec130d5..3d612f5549 100644 --- a/modules/git/batch_reader.go +++ b/modules/git/batch_reader.go @@ -5,320 +5,53 @@ package git import ( "bufio" - "bytes" - "context" - "io" - "math" - "strconv" - "strings" + "errors" - "code.gitea.io/gitea/modules/git/gitcmd" - "code.gitea.io/gitea/modules/log" - - "github.com/djherbis/buffer" - "github.com/djherbis/nio/v3" + "code.gitea.io/gitea/modules/git/catfile" ) -// WriteCloserError wraps an io.WriteCloser with an additional CloseWithError function -type WriteCloserError interface { - io.WriteCloser - CloseWithError(err error) error -} - -// ensureValidGitRepository runs git rev-parse in the repository path - thus ensuring that the repository is a valid repository. -// Run before opening git cat-file. -// This is needed otherwise the git cat-file will hang for invalid repositories. -func ensureValidGitRepository(ctx context.Context, repoPath string) error { - stderr := strings.Builder{} - err := gitcmd.NewCommand("rev-parse"). - WithDir(repoPath). - WithStderr(&stderr). - Run(ctx) - if err != nil { - return gitcmd.ConcatenateError(err, (&stderr).String()) - } - return nil -} - -// catFileBatchCheck opens git cat-file --batch-check in the provided repo and returns a stdin pipe, a stdout reader and cancel function -func catFileBatchCheck(ctx context.Context, repoPath string) (WriteCloserError, *bufio.Reader, func()) { - batchStdinReader, batchStdinWriter := io.Pipe() - batchStdoutReader, batchStdoutWriter := io.Pipe() - ctx, ctxCancel := context.WithCancel(ctx) - closed := make(chan struct{}) - cancel := func() { - ctxCancel() - _ = batchStdoutReader.Close() - _ = batchStdinWriter.Close() - <-closed - } - - // Ensure cancel is called as soon as the provided context is cancelled - go func() { - <-ctx.Done() - cancel() - }() - - go func() { - stderr := strings.Builder{} - err := gitcmd.NewCommand("cat-file", "--batch-check"). - WithDir(repoPath). - WithStdin(batchStdinReader). - WithStdout(batchStdoutWriter). - WithStderr(&stderr). - WithUseContextTimeout(true). - Run(ctx) - if err != nil { - _ = batchStdoutWriter.CloseWithError(gitcmd.ConcatenateError(err, (&stderr).String())) - _ = batchStdinReader.CloseWithError(gitcmd.ConcatenateError(err, (&stderr).String())) - } else { - _ = batchStdoutWriter.Close() - _ = batchStdinReader.Close() - } - close(closed) - }() - - // For simplicities sake we'll use a buffered reader to read from the cat-file --batch-check - batchReader := bufio.NewReader(batchStdoutReader) - - return batchStdinWriter, batchReader, cancel -} - -// catFileBatch opens git cat-file --batch in the provided repo and returns a stdin pipe, a stdout reader and cancel function -func catFileBatch(ctx context.Context, repoPath string) (WriteCloserError, *bufio.Reader, func()) { - // We often want to feed the commits in order into cat-file --batch, followed by their trees and sub trees as necessary. - // so let's create a batch stdin and stdout - batchStdinReader, batchStdinWriter := io.Pipe() - batchStdoutReader, batchStdoutWriter := nio.Pipe(buffer.New(32 * 1024)) - ctx, ctxCancel := context.WithCancel(ctx) - closed := make(chan struct{}) - cancel := func() { - ctxCancel() - _ = batchStdinWriter.Close() - _ = batchStdoutReader.Close() - <-closed - } - - // Ensure cancel is called as soon as the provided context is cancelled - go func() { - <-ctx.Done() - cancel() - }() - - go func() { - stderr := strings.Builder{} - err := gitcmd.NewCommand("cat-file", "--batch"). - WithDir(repoPath). - WithStdin(batchStdinReader). - WithStdout(batchStdoutWriter). - WithStderr(&stderr). - WithUseContextTimeout(true). - Run(ctx) - if err != nil { - _ = batchStdoutWriter.CloseWithError(gitcmd.ConcatenateError(err, (&stderr).String())) - _ = batchStdinReader.CloseWithError(gitcmd.ConcatenateError(err, (&stderr).String())) - } else { - _ = batchStdoutWriter.Close() - _ = batchStdinReader.Close() - } - close(closed) - }() - - // For simplicities sake we'll us a buffered reader to read from the cat-file --batch - batchReader := bufio.NewReaderSize(batchStdoutReader, 32*1024) - - return batchStdinWriter, batchReader, cancel -} - -// ReadBatchLine reads the header line from cat-file --batch -// We expect: SP SP LF -// then leaving the rest of the stream " LF" to be read +// ReadBatchLine reads the header line from cat-file --batch while preserving the traditional return signature. func ReadBatchLine(rd *bufio.Reader) (sha []byte, typ string, size int64, err error) { - typ, err = rd.ReadString('\n') - if err != nil { - return sha, typ, size, err - } - if len(typ) == 1 { - typ, err = rd.ReadString('\n') - if err != nil { - return sha, typ, size, err - } - } - idx := strings.IndexByte(typ, ' ') - if idx < 0 { - log.Debug("missing space typ: %s", typ) - return sha, typ, size, ErrNotExist{ID: string(sha)} - } - sha = []byte(typ[:idx]) - typ = typ[idx+1:] - - idx = strings.IndexByte(typ, ' ') - if idx < 0 { - return sha, typ, size, ErrNotExist{ID: string(sha)} - } - - sizeStr := typ[idx+1 : len(typ)-1] - typ = typ[:idx] - - size, err = strconv.ParseInt(sizeStr, 10, 64) - return sha, typ, size, err + sha, typ, size, err = catfile.ReadBatchLine(rd) + return sha, typ, size, convertCatfileError(err, sha) } // ReadTagObjectID reads a tag object ID hash from a cat-file --batch stream, throwing away the rest of the stream. func ReadTagObjectID(rd *bufio.Reader, size int64) (string, error) { - var id string - var n int64 -headerLoop: - for { - line, err := rd.ReadBytes('\n') - if err != nil { - return "", err - } - n += int64(len(line)) - idx := bytes.Index(line, []byte{' '}) - if idx < 0 { - continue - } - - if string(line[:idx]) == "object" { - id = string(line[idx+1 : len(line)-1]) - break headerLoop - } - } - - // Discard the rest of the tag - return id, DiscardFull(rd, size-n+1) + return catfile.ReadTagObjectID(rd, size) } // ReadTreeID reads a tree ID from a cat-file --batch stream, throwing away the rest of the stream. func ReadTreeID(rd *bufio.Reader, size int64) (string, error) { - var id string - var n int64 -headerLoop: - for { - line, err := rd.ReadBytes('\n') - if err != nil { - return "", err - } - n += int64(len(line)) - idx := bytes.Index(line, []byte{' '}) - if idx < 0 { - continue - } - - if string(line[:idx]) == "tree" { - id = string(line[idx+1 : len(line)-1]) - break headerLoop - } - } - - // Discard the rest of the commit - return id, DiscardFull(rd, size-n+1) + return catfile.ReadTreeID(rd, size) } -// git tree files are a list: -// SP NUL -// -// Unfortunately this 20-byte notation is somewhat in conflict to all other git tools -// Therefore we need some method to convert these binary hashes to hex hashes - -// constant hextable to help quickly convert between binary and hex representation -const hextable = "0123456789abcdef" - -// BinToHexHeash converts a binary Hash into a hex encoded one. Input and output can be the -// same byte slice to support in place conversion without allocations. -// This is at least 100x quicker that hex.EncodeToString +// BinToHex converts a binary hash into a hex encoded one. func BinToHex(objectFormat ObjectFormat, sha, out []byte) []byte { - for i := objectFormat.FullLength()/2 - 1; i >= 0; i-- { - v := sha[i] - vhi, vlo := v>>4, v&0x0f - shi, slo := hextable[vhi], hextable[vlo] - out[i*2], out[i*2+1] = shi, slo - } - return out + return catfile.BinToHex(objectFormat, sha, out) } -// ParseCatFileTreeLine reads an entry from a tree in a cat-file --batch stream -// This carefully avoids allocations - except where fnameBuf is too small. -// It is recommended therefore to pass in an fnameBuf large enough to avoid almost all allocations -// -// Each line is composed of: -// SP NUL -// -// We don't attempt to convert the raw HASH to save a lot of time +// ParseCatFileTreeLine reads an entry from a tree in a cat-file --batch stream. func ParseCatFileTreeLine(objectFormat ObjectFormat, rd *bufio.Reader, modeBuf, fnameBuf, shaBuf []byte) (mode, fname, sha []byte, n int, err error) { - var readBytes []byte - - // Read the Mode & fname - readBytes, err = rd.ReadSlice('\x00') - if err != nil { - return mode, fname, sha, n, err - } - idx := bytes.IndexByte(readBytes, ' ') - if idx < 0 { - log.Debug("missing space in readBytes ParseCatFileTreeLine: %s", readBytes) - return mode, fname, sha, n, &ErrNotExist{} - } - - n += idx + 1 - copy(modeBuf, readBytes[:idx]) - if len(modeBuf) >= idx { - modeBuf = modeBuf[:idx] - } else { - modeBuf = append(modeBuf, readBytes[len(modeBuf):idx]...) - } - mode = modeBuf - - readBytes = readBytes[idx+1:] - - // Deal with the fname - copy(fnameBuf, readBytes) - if len(fnameBuf) > len(readBytes) { - fnameBuf = fnameBuf[:len(readBytes)] - } else { - fnameBuf = append(fnameBuf, readBytes[len(fnameBuf):]...) - } - for err == bufio.ErrBufferFull { - readBytes, err = rd.ReadSlice('\x00') - fnameBuf = append(fnameBuf, readBytes...) - } - n += len(fnameBuf) - if err != nil { - return mode, fname, sha, n, err - } - fnameBuf = fnameBuf[:len(fnameBuf)-1] - fname = fnameBuf - - // Deal with the binary hash - idx = 0 - length := objectFormat.FullLength() / 2 - for idx < length { - var read int - read, err = rd.Read(shaBuf[idx:length]) - n += read - if err != nil { - return mode, fname, sha, n, err - } - idx += read - } - sha = shaBuf - return mode, fname, sha, n, err + mode, fname, sha, n, err = catfile.ParseCatFileTreeLine(objectFormat, rd, modeBuf, fnameBuf, shaBuf) + return mode, fname, sha, n, convertCatfileError(err, nil) } +// DiscardFull discards the requested number of bytes from the buffered reader. func DiscardFull(rd *bufio.Reader, discard int64) error { - if discard > math.MaxInt32 { - n, err := rd.Discard(math.MaxInt32) - discard -= int64(n) - if err != nil { - return err - } - } - for discard > 0 { - n, err := rd.Discard(int(discard)) - discard -= int64(n) - if err != nil { - return err - } - } - return nil + return catfile.DiscardFull(rd, discard) +} + +func convertCatfileError(err error, defaultID []byte) error { + if err == nil { + return nil + } + var notFound catfile.ErrObjectNotFound + if errors.As(err, ¬Found) { + if notFound.ID == "" && len(defaultID) > 0 { + notFound.ID = string(defaultID) + } + return ErrNotExist{ID: notFound.ID} + } + return err } diff --git a/modules/git/blob_nogogit.go b/modules/git/blob_nogogit.go index af3ce376d6..88e2be792b 100644 --- a/modules/git/blob_nogogit.go +++ b/modules/git/blob_nogogit.go @@ -26,12 +26,13 @@ type Blob struct { // DataAsync gets a ReadCloser for the contents of a blob without reading it all. // Calling the Close function on the result will discard all unread output. func (b *Blob) DataAsync() (io.ReadCloser, error) { - wr, rd, cancel, err := b.repo.CatFileBatch(b.repo.Ctx) + batch, cancel, err := b.repo.CatFileBatch(b.repo.Ctx) if err != nil { return nil, err } - _, err = wr.Write([]byte(b.ID.String() + "\n")) + rd := batch.Reader() + _, err = batch.Writer().Write([]byte(b.ID.String() + "\n")) if err != nil { cancel() return nil, err @@ -67,18 +68,18 @@ func (b *Blob) Size() int64 { return b.size } - wr, rd, cancel, err := b.repo.CatFileBatchCheck(b.repo.Ctx) + batch, cancel, err := b.repo.CatFileBatchCheck(b.repo.Ctx) if err != nil { log.Debug("error whilst reading size for %s in %s. Error: %v", b.ID.String(), b.repo.Path, err) return 0 } defer cancel() - _, err = wr.Write([]byte(b.ID.String() + "\n")) + _, err = batch.Writer().Write([]byte(b.ID.String() + "\n")) if err != nil { log.Debug("error whilst reading size for %s in %s. Error: %v", b.ID.String(), b.repo.Path, err) return 0 } - _, _, b.size, err = ReadBatchLine(rd) + _, _, b.size, err = ReadBatchLine(batch.Reader()) if err != nil { log.Debug("error whilst reading size for %s in %s. Error: %v", b.ID.String(), b.repo.Path, err) return 0 diff --git a/modules/git/catfile/batch.go b/modules/git/catfile/batch.go new file mode 100644 index 0000000000..1facb8946e --- /dev/null +++ b/modules/git/catfile/batch.go @@ -0,0 +1,178 @@ +// Copyright 2024 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package catfile + +import ( + "bufio" + "context" + "io" + "strings" + + "code.gitea.io/gitea/modules/git/gitcmd" + + "github.com/djherbis/buffer" + "github.com/djherbis/nio/v3" +) + +// WriteCloserError wraps an io.WriteCloser with an additional CloseWithError function +type WriteCloserError interface { + io.WriteCloser + CloseWithError(err error) error +} + +type Batch interface { + Writer() WriteCloserError + Reader() *bufio.Reader + Close() +} + +// batch represents an active `git cat-file --batch` or `--batch-check` invocation +// paired with the pipes that feed/read from it. Call Close to release resources. +type batch struct { + cancel context.CancelFunc + reader *bufio.Reader + writer WriteCloserError +} + +// NewBatch creates a new cat-file --batch process for the provided repository path. +// The returned Batch must be closed once the caller has finished with it. +func NewBatch(ctx context.Context, repoPath string) (Batch, error) { + if err := EnsureValidGitRepository(ctx, repoPath); err != nil { + return nil, err + } + + var batch batch + batch.writer, batch.reader, batch.cancel = catFileBatch(ctx, repoPath) + return &batch, nil +} + +// NewBatchCheck creates a cat-file --batch-check process for the provided repository path. +// The returned Batch must be closed once the caller has finished with it. +func NewBatchCheck(ctx context.Context, repoPath string) (Batch, error) { + if err := EnsureValidGitRepository(ctx, repoPath); err != nil { + return nil, err + } + + var check batch + check.writer, check.reader, check.cancel = catFileBatchCheck(ctx, repoPath) + return &check, nil +} + +func (b *batch) Writer() WriteCloserError { + return b.writer +} + +func (b *batch) Reader() *bufio.Reader { + return b.reader +} + +// Close stops the underlying git cat-file process and releases held resources. +func (b *batch) Close() { + if b == nil || b.cancel == nil { + return + } + b.cancel() + b.reader = nil + b.writer = nil + b.cancel = nil +} + +// EnsureValidGitRepository runs `git rev-parse` in the repository path to make sure +// the directory is a valid git repository. This avoids git cat-file hanging indefinitely +// when invoked in invalid paths. +func EnsureValidGitRepository(ctx context.Context, repoPath string) error { + stder := strings.Builder{} + err := gitcmd.NewCommand("rev-parse"). + WithDir(repoPath). + WithStderr(&stder). + Run(ctx) + if err != nil { + return gitcmd.ConcatenateError(err, stder.String()) + } + return nil +} + +// catFileBatch opens git cat-file --batch in the provided repo and returns a stdin pipe, +// a stdout reader and a cancel function. +func catFileBatch(ctx context.Context, repoPath string) (WriteCloserError, *bufio.Reader, func()) { + batchStdinReader, batchStdinWriter := io.Pipe() + batchStdoutReader, batchStdoutWriter := nio.Pipe(buffer.New(32 * 1024)) + ctx, ctxCancel := context.WithCancel(ctx) + closed := make(chan struct{}) + cancel := func() { + ctxCancel() + _ = batchStdinWriter.Close() + _ = batchStdoutReader.Close() + <-closed + } + + go func() { + <-ctx.Done() + cancel() + }() + + go func() { + stder := strings.Builder{} + err := gitcmd.NewCommand("cat-file", "--batch"). + WithDir(repoPath). + WithStdin(batchStdinReader). + WithStdout(batchStdoutWriter). + WithStderr(&stder). + WithUseContextTimeout(true). + Run(ctx) + if err != nil { + _ = batchStdoutWriter.CloseWithError(gitcmd.ConcatenateError(err, stder.String())) + _ = batchStdinReader.CloseWithError(gitcmd.ConcatenateError(err, stder.String())) + } else { + _ = batchStdoutWriter.Close() + _ = batchStdinReader.Close() + } + close(closed) + }() + + batchReader := bufio.NewReaderSize(batchStdoutReader, 32*1024) + return batchStdinWriter, batchReader, cancel +} + +// catFileBatchCheck opens git cat-file --batch-check in the provided repo and returns a stdin pipe, +// a stdout reader and cancel function. +func catFileBatchCheck(ctx context.Context, repoPath string) (WriteCloserError, *bufio.Reader, func()) { + batchStdinReader, batchStdinWriter := io.Pipe() + batchStdoutReader, batchStdoutWriter := io.Pipe() + ctx, ctxCancel := context.WithCancel(ctx) + closed := make(chan struct{}) + cancel := func() { + ctxCancel() + _ = batchStdoutReader.Close() + _ = batchStdinWriter.Close() + <-closed + } + + go func() { + <-ctx.Done() + cancel() + }() + + go func() { + stder := strings.Builder{} + err := gitcmd.NewCommand("cat-file", "--batch-check"). + WithDir(repoPath). + WithStdin(batchStdinReader). + WithStdout(batchStdoutWriter). + WithStderr(&stder). + WithUseContextTimeout(true). + Run(ctx) + if err != nil { + _ = batchStdoutWriter.CloseWithError(gitcmd.ConcatenateError(err, stder.String())) + _ = batchStdinReader.CloseWithError(gitcmd.ConcatenateError(err, stder.String())) + } else { + _ = batchStdoutWriter.Close() + _ = batchStdinReader.Close() + } + close(closed) + }() + + batchReader := bufio.NewReader(batchStdoutReader) + return batchStdinWriter, batchReader, cancel +} diff --git a/modules/git/catfile/reader.go b/modules/git/catfile/reader.go new file mode 100644 index 0000000000..1785cc4cc0 --- /dev/null +++ b/modules/git/catfile/reader.go @@ -0,0 +1,211 @@ +// Copyright 2020 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package catfile + +import ( + "bufio" + "bytes" + "errors" + "fmt" + "math" + "strconv" + "strings" + + "code.gitea.io/gitea/modules/log" +) + +// ErrObjectNotFound indicates that the requested object could not be read from cat-file +type ErrObjectNotFound struct { + ID string +} + +func (err ErrObjectNotFound) Error() string { + return fmt.Sprintf("catfile: object does not exist [id: %s]", err.ID) +} + +// IsErrObjectNotFound reports whether err is an ErrObjectNotFound +func IsErrObjectNotFound(err error) bool { + var target ErrObjectNotFound + return errors.As(err, &target) +} + +// ObjectFormat abstracts the minimal information needed from git.ObjectFormat. +type ObjectFormat interface { + FullLength() int +} + +// ReadBatchLine reads the header line from cat-file --batch. It expects the format +// " SP SP LF" and leaves the reader positioned at the start of +// the object contents (which must be fully consumed by the caller). +func ReadBatchLine(rd *bufio.Reader) (sha []byte, typ string, size int64, err error) { + typ, err = rd.ReadString('\n') + if err != nil { + return sha, typ, size, err + } + if len(typ) == 1 { + typ, err = rd.ReadString('\n') + if err != nil { + return sha, typ, size, err + } + } + idx := strings.IndexByte(typ, ' ') + if idx < 0 { + return sha, typ, size, ErrObjectNotFound{} + } + sha = []byte(typ[:idx]) + typ = typ[idx+1:] + + idx = strings.IndexByte(typ, ' ') + if idx < 0 { + return sha, typ, size, ErrObjectNotFound{ID: string(sha)} + } + + sizeStr := typ[idx+1 : len(typ)-1] + typ = typ[:idx] + + size, err = strconv.ParseInt(sizeStr, 10, 64) + return sha, typ, size, err +} + +// ReadTagObjectID reads a tag object ID hash from a cat-file --batch stream, throwing away the rest. +func ReadTagObjectID(rd *bufio.Reader, size int64) (string, error) { + var id string + var n int64 +headerLoop: + for { + line, err := rd.ReadBytes('\n') + if err != nil { + return "", err + } + n += int64(len(line)) + idx := bytes.Index(line, []byte{' '}) + if idx < 0 { + continue + } + + if string(line[:idx]) == "object" { + id = string(line[idx+1 : len(line)-1]) + break headerLoop + } + } + + return id, DiscardFull(rd, size-n+1) +} + +// ReadTreeID reads a tree ID from a cat-file --batch stream, throwing away the rest of the commit content. +func ReadTreeID(rd *bufio.Reader, size int64) (string, error) { + var id string + var n int64 +headerLoop: + for { + line, err := rd.ReadBytes('\n') + if err != nil { + return "", err + } + n += int64(len(line)) + idx := bytes.Index(line, []byte{' '}) + if idx < 0 { + continue + } + + if string(line[:idx]) == "tree" { + id = string(line[idx+1 : len(line)-1]) + break headerLoop + } + } + + return id, DiscardFull(rd, size-n+1) +} + +// hextable helps quickly convert between binary and hex representation +const hextable = "0123456789abcdef" + +// BinToHex converts a binary hash into a hex encoded one. Input and output can be the +// same byte slice to support in-place conversion without allocations. +func BinToHex(objectFormat ObjectFormat, sha, out []byte) []byte { + for i := objectFormat.FullLength()/2 - 1; i >= 0; i-- { + v := sha[i] + vhi, vlo := v>>4, v&0x0f + shi, slo := hextable[vhi], hextable[vlo] + out[i*2], out[i*2+1] = shi, slo + } + return out +} + +// ParseCatFileTreeLine reads an entry from a tree in a cat-file --batch stream and avoids allocations +// where possible. Each line is composed of: +// SP NUL +func ParseCatFileTreeLine(objectFormat ObjectFormat, rd *bufio.Reader, modeBuf, fnameBuf, shaBuf []byte) (mode, fname, sha []byte, n int, err error) { + var readBytes []byte + + readBytes, err = rd.ReadSlice('\x00') + if err != nil { + return mode, fname, sha, n, err + } + idx := bytes.IndexByte(readBytes, ' ') + if idx < 0 { + log.Debug("missing space in readBytes ParseCatFileTreeLine: %s", readBytes) + return mode, fname, sha, n, ErrObjectNotFound{} + } + + n += idx + 1 + copy(modeBuf, readBytes[:idx]) + if len(modeBuf) >= idx { + modeBuf = modeBuf[:idx] + } else { + modeBuf = append(modeBuf, readBytes[len(modeBuf):idx]...) + } + mode = modeBuf + + readBytes = readBytes[idx+1:] + copy(fnameBuf, readBytes) + if len(fnameBuf) > len(readBytes) { + fnameBuf = fnameBuf[:len(readBytes)] + } else { + fnameBuf = append(fnameBuf, readBytes[len(fnameBuf):]...) + } + for err == bufio.ErrBufferFull { + readBytes, err = rd.ReadSlice('\x00') + fnameBuf = append(fnameBuf, readBytes...) + } + n += len(fnameBuf) + if err != nil { + return mode, fname, sha, n, err + } + fnameBuf = fnameBuf[:len(fnameBuf)-1] + fname = fnameBuf + + idx = 0 + length := objectFormat.FullLength() / 2 + for idx < length { + var read int + read, err = rd.Read(shaBuf[idx:length]) + n += read + if err != nil { + return mode, fname, sha, n, err + } + idx += read + } + sha = shaBuf + return mode, fname, sha, n, err +} + +// DiscardFull discards the requested amount of bytes from the buffered reader regardless of its internal limit. +func DiscardFull(rd *bufio.Reader, discard int64) error { + if discard > math.MaxInt32 { + n, err := rd.Discard(math.MaxInt32) + discard -= int64(n) + if err != nil { + return err + } + } + for discard > 0 { + n, err := rd.Discard(int(discard)) + discard -= int64(n) + if err != nil { + return err + } + } + return nil +} diff --git a/modules/git/languagestats/language_stats_nogogit.go b/modules/git/languagestats/language_stats_nogogit.go index 94cf9fff8c..da291ae848 100644 --- a/modules/git/languagestats/language_stats_nogogit.go +++ b/modules/git/languagestats/language_stats_nogogit.go @@ -22,20 +22,21 @@ import ( func GetLanguageStats(repo *git.Repository, commitID string) (map[string]int64, error) { // We will feed the commit IDs in order into cat-file --batch, followed by blobs as necessary. // so let's create a batch stdin and stdout - batchStdinWriter, batchReader, cancel, err := repo.CatFileBatch(repo.Ctx) + batch, cancel, err := repo.CatFileBatch(repo.Ctx) if err != nil { return nil, err } defer cancel() writeID := func(id string) error { - _, err := batchStdinWriter.Write([]byte(id + "\n")) + _, err := batch.Writer().Write([]byte(id + "\n")) return err } if err := writeID(commitID); err != nil { return nil, err } + batchReader := batch.Reader() shaBytes, typ, size, err := git.ReadBatchLine(batchReader) if typ != "commit" { log.Debug("Unable to get commit for: %s. Err: %v", commitID, err) diff --git a/modules/git/pipeline/lfs_nogogit.go b/modules/git/pipeline/lfs_nogogit.go index 4881a2be64..6f1a860c1d 100644 --- a/modules/git/pipeline/lfs_nogogit.go +++ b/modules/git/pipeline/lfs_nogogit.go @@ -47,12 +47,15 @@ func FindLFSFile(repo *git.Repository, objectID git.ObjectID) ([]*LFSResult, err // Next feed the commits in order into cat-file --batch, followed by their trees and sub trees as necessary. // so let's create a batch stdin and stdout - batchStdinWriter, batchReader, cancel, err := repo.CatFileBatch(repo.Ctx) + batch, cancel, err := repo.CatFileBatch(repo.Ctx) if err != nil { return nil, err } defer cancel() + batchStdinWriter := batch.Writer() + batchReader := batch.Reader() + // We'll use a scanner for the revList because it's simpler than a bufio.Reader scan := bufio.NewScanner(revListReader) trees := [][]byte{} diff --git a/modules/git/repo_base_nogogit.go b/modules/git/repo_base_nogogit.go index 17c71da5ef..97a43b90fd 100644 --- a/modules/git/repo_base_nogogit.go +++ b/modules/git/repo_base_nogogit.go @@ -7,10 +7,10 @@ package git import ( - "bufio" "context" "path/filepath" + "code.gitea.io/gitea/modules/git/catfile" "code.gitea.io/gitea/modules/log" "code.gitea.io/gitea/modules/util" ) @@ -24,10 +24,10 @@ type Repository struct { tagCache *ObjectCache[*Tag] batchInUse bool - batch *Batch + batch catfile.Batch checkInUse bool - check *Batch + check catfile.Batch Ctx context.Context LastCommitCache *LastCommitCache @@ -57,53 +57,53 @@ func OpenRepository(ctx context.Context, repoPath string) (*Repository, error) { } // CatFileBatch obtains a CatFileBatch for this repository -func (repo *Repository) CatFileBatch(ctx context.Context) (WriteCloserError, *bufio.Reader, func(), error) { +func (repo *Repository) CatFileBatch(ctx context.Context) (catfile.Batch, func(), error) { if repo.batch == nil { var err error - repo.batch, err = NewBatch(ctx, repo.Path) + repo.batch, err = catfile.NewBatch(ctx, repo.Path) if err != nil { - return nil, nil, nil, err + return nil, nil, err } } if !repo.batchInUse { repo.batchInUse = true - return repo.batch.Writer, repo.batch.Reader, func() { + return repo.batch, func() { repo.batchInUse = false }, nil } log.Debug("Opening temporary cat file batch for: %s", repo.Path) - tempBatch, err := NewBatch(ctx, repo.Path) + tempBatch, err := catfile.NewBatch(ctx, repo.Path) if err != nil { - return nil, nil, nil, err + return nil, nil, err } - return tempBatch.Writer, tempBatch.Reader, tempBatch.Close, nil + return tempBatch, tempBatch.Close, nil } // CatFileBatchCheck obtains a CatFileBatchCheck for this repository -func (repo *Repository) CatFileBatchCheck(ctx context.Context) (WriteCloserError, *bufio.Reader, func(), error) { +func (repo *Repository) CatFileBatchCheck(ctx context.Context) (catfile.Batch, func(), error) { if repo.check == nil { var err error - repo.check, err = NewBatchCheck(ctx, repo.Path) + repo.check, err = catfile.NewBatchCheck(ctx, repo.Path) if err != nil { - return nil, nil, nil, err + return nil, nil, err } } if !repo.checkInUse { repo.checkInUse = true - return repo.check.Writer, repo.check.Reader, func() { + return repo.check, func() { repo.checkInUse = false }, nil } log.Debug("Opening temporary cat file batch-check for: %s", repo.Path) - tempBatchCheck, err := NewBatchCheck(ctx, repo.Path) + tempBatchCheck, err := catfile.NewBatchCheck(ctx, repo.Path) if err != nil { - return nil, nil, nil, err + return nil, nil, err } - return tempBatchCheck.Writer, tempBatchCheck.Reader, tempBatchCheck.Close, nil + return tempBatchCheck, tempBatchCheck.Close, nil } func (repo *Repository) Close() error { diff --git a/modules/git/repo_branch_nogogit.go b/modules/git/repo_branch_nogogit.go index f1b26b06ab..09873fb2c6 100644 --- a/modules/git/repo_branch_nogogit.go +++ b/modules/git/repo_branch_nogogit.go @@ -23,18 +23,18 @@ func (repo *Repository) IsObjectExist(name string) bool { return false } - wr, rd, cancel, err := repo.CatFileBatchCheck(repo.Ctx) + batch, cancel, err := repo.CatFileBatchCheck(repo.Ctx) if err != nil { log.Debug("Error writing to CatFileBatchCheck %v", err) return false } defer cancel() - _, err = wr.Write([]byte(name + "\n")) + _, err = batch.Writer().Write([]byte(name + "\n")) if err != nil { log.Debug("Error writing to CatFileBatchCheck %v", err) return false } - sha, _, _, err := ReadBatchLine(rd) + sha, _, _, err := ReadBatchLine(batch.Reader()) return err == nil && bytes.HasPrefix(sha, []byte(strings.TrimSpace(name))) } @@ -44,18 +44,18 @@ func (repo *Repository) IsReferenceExist(name string) bool { return false } - wr, rd, cancel, err := repo.CatFileBatchCheck(repo.Ctx) + batch, cancel, err := repo.CatFileBatchCheck(repo.Ctx) if err != nil { log.Debug("Error writing to CatFileBatchCheck %v", err) return false } defer cancel() - _, err = wr.Write([]byte(name + "\n")) + _, err = batch.Writer().Write([]byte(name + "\n")) if err != nil { log.Debug("Error writing to CatFileBatchCheck %v", err) return false } - _, _, _, err = ReadBatchLine(rd) + _, _, _, err = ReadBatchLine(batch.Reader()) return err == nil } diff --git a/modules/git/repo_commit_nogogit.go b/modules/git/repo_commit_nogogit.go index 3f27833fa6..a3d728eb6d 100644 --- a/modules/git/repo_commit_nogogit.go +++ b/modules/git/repo_commit_nogogit.go @@ -6,11 +6,11 @@ package git import ( - "bufio" "errors" "io" "strings" + "code.gitea.io/gitea/modules/git/catfile" "code.gitea.io/gitea/modules/git/gitcmd" "code.gitea.io/gitea/modules/log" ) @@ -37,16 +37,16 @@ func (repo *Repository) ResolveReference(name string) (string, error) { // GetRefCommitID returns the last commit ID string of given reference (branch or tag). func (repo *Repository) GetRefCommitID(name string) (string, error) { - wr, rd, cancel, err := repo.CatFileBatchCheck(repo.Ctx) + batch, cancel, err := repo.CatFileBatchCheck(repo.Ctx) if err != nil { return "", err } defer cancel() - _, err = wr.Write([]byte(name + "\n")) + _, err = batch.Writer().Write([]byte(name + "\n")) if err != nil { return "", err } - shaBs, _, _, err := ReadBatchLine(rd) + shaBs, _, _, err := ReadBatchLine(batch.Reader()) if IsErrNotExist(err) { return "", ErrNotExist{name, ""} } @@ -56,7 +56,7 @@ func (repo *Repository) GetRefCommitID(name string) (string, error) { // IsCommitExist returns true if given commit exists in current repository. func (repo *Repository) IsCommitExist(name string) bool { - if err := ensureValidGitRepository(repo.Ctx, repo.Path); err != nil { + if err := catfile.EnsureValidGitRepository(repo.Ctx, repo.Path); err != nil { log.Error("IsCommitExist: %v", err) return false } @@ -68,18 +68,19 @@ func (repo *Repository) IsCommitExist(name string) bool { } func (repo *Repository) getCommit(id ObjectID) (*Commit, error) { - wr, rd, cancel, err := repo.CatFileBatch(repo.Ctx) + batch, cancel, err := repo.CatFileBatch(repo.Ctx) if err != nil { return nil, err } defer cancel() - _, _ = wr.Write([]byte(id.String() + "\n")) + _, _ = batch.Writer().Write([]byte(id.String() + "\n")) - return repo.getCommitFromBatchReader(wr, rd, id) + return repo.getCommitFromBatchReader(batch, id) } -func (repo *Repository) getCommitFromBatchReader(wr WriteCloserError, rd *bufio.Reader, id ObjectID) (*Commit, error) { +func (repo *Repository) getCommitFromBatchReader(batch catfile.Batch, id ObjectID) (*Commit, error) { + rd := batch.Reader() _, typ, size, err := ReadBatchLine(rd) if err != nil { if errors.Is(err, io.EOF) || IsErrNotExist(err) { @@ -107,11 +108,11 @@ func (repo *Repository) getCommitFromBatchReader(wr WriteCloserError, rd *bufio. return nil, err } - if _, err := wr.Write([]byte(tag.Object.String() + "\n")); err != nil { + if _, err := batch.Writer().Write([]byte(tag.Object.String() + "\n")); err != nil { return nil, err } - commit, err := repo.getCommitFromBatchReader(wr, rd, tag.Object) + commit, err := repo.getCommitFromBatchReader(batch, tag.Object) if err != nil { return nil, err } @@ -152,16 +153,16 @@ func (repo *Repository) ConvertToGitID(commitID string) (ObjectID, error) { } } - wr, rd, cancel, err := repo.CatFileBatchCheck(repo.Ctx) + batch, cancel, err := repo.CatFileBatchCheck(repo.Ctx) if err != nil { return nil, err } defer cancel() - _, err = wr.Write([]byte(commitID + "\n")) + _, err = batch.Writer().Write([]byte(commitID + "\n")) if err != nil { return nil, err } - sha, _, _, err := ReadBatchLine(rd) + sha, _, _, err := ReadBatchLine(batch.Reader()) if err != nil { if IsErrNotExist(err) { return nil, ErrNotExist{commitID, ""} diff --git a/modules/git/repo_tag_nogogit.go b/modules/git/repo_tag_nogogit.go index 5f79b68a9a..88d9edcbd8 100644 --- a/modules/git/repo_tag_nogogit.go +++ b/modules/git/repo_tag_nogogit.go @@ -24,16 +24,16 @@ func (repo *Repository) IsTagExist(name string) bool { // GetTagType gets the type of the tag, either commit (simple) or tag (annotated) func (repo *Repository) GetTagType(id ObjectID) (string, error) { - wr, rd, cancel, err := repo.CatFileBatchCheck(repo.Ctx) + batch, cancel, err := repo.CatFileBatchCheck(repo.Ctx) if err != nil { return "", err } defer cancel() - _, err = wr.Write([]byte(id.String() + "\n")) + _, err = batch.Writer().Write([]byte(id.String() + "\n")) if err != nil { return "", err } - _, typ, _, err := ReadBatchLine(rd) + _, typ, _, err := ReadBatchLine(batch.Reader()) if err != nil { if IsErrNotExist(err) { return "", ErrNotExist{ID: id.String()} @@ -88,13 +88,14 @@ func (repo *Repository) getTag(tagID ObjectID, name string) (*Tag, error) { } // The tag is an annotated tag with a message. - wr, rd, cancel, err := repo.CatFileBatch(repo.Ctx) + batch, cancel, err := repo.CatFileBatch(repo.Ctx) if err != nil { return nil, err } defer cancel() - if _, err := wr.Write([]byte(tagID.String() + "\n")); err != nil { + rd := batch.Reader() + if _, err := batch.Writer().Write([]byte(tagID.String() + "\n")); err != nil { return nil, err } _, typ, size, err := ReadBatchLine(rd) diff --git a/modules/git/repo_tree_nogogit.go b/modules/git/repo_tree_nogogit.go index 1954f85162..e6e2ee9fa0 100644 --- a/modules/git/repo_tree_nogogit.go +++ b/modules/git/repo_tree_nogogit.go @@ -10,12 +10,14 @@ import ( ) func (repo *Repository) getTree(id ObjectID) (*Tree, error) { - wr, rd, cancel, err := repo.CatFileBatch(repo.Ctx) + batch, cancel, err := repo.CatFileBatch(repo.Ctx) if err != nil { return nil, err } defer cancel() + wr := batch.Writer() + rd := batch.Reader() _, _ = wr.Write([]byte(id.String() + "\n")) // ignore the SHA @@ -39,7 +41,7 @@ func (repo *Repository) getTree(id ObjectID) (*Tree, error) { if _, err := wr.Write([]byte(tag.Object.String() + "\n")); err != nil { return nil, err } - commit, err := repo.getCommitFromBatchReader(wr, rd, tag.Object) + commit, err := repo.getCommitFromBatchReader(batch, tag.Object) if err != nil { return nil, err } diff --git a/modules/git/tree_entry_nogogit.go b/modules/git/tree_entry_nogogit.go index fd2f3c567f..0ea7aeed9d 100644 --- a/modules/git/tree_entry_nogogit.go +++ b/modules/git/tree_entry_nogogit.go @@ -15,18 +15,18 @@ func (te *TreeEntry) Size() int64 { return te.size } - wr, rd, cancel, err := te.ptree.repo.CatFileBatchCheck(te.ptree.repo.Ctx) + batch, cancel, err := te.ptree.repo.CatFileBatchCheck(te.ptree.repo.Ctx) if err != nil { log.Debug("error whilst reading size for %s in %s. Error: %v", te.ID.String(), te.ptree.repo.Path, err) return 0 } defer cancel() - _, err = wr.Write([]byte(te.ID.String() + "\n")) + _, err = batch.Writer().Write([]byte(te.ID.String() + "\n")) if err != nil { log.Debug("error whilst reading size for %s in %s. Error: %v", te.ID.String(), te.ptree.repo.Path, err) return 0 } - _, _, te.size, err = ReadBatchLine(rd) + _, _, te.size, err = ReadBatchLine(batch.Reader()) if err != nil { log.Debug("error whilst reading size for %s in %s. Error: %v", te.ID.String(), te.ptree.repo.Path, err) return 0 diff --git a/modules/git/tree_nogogit.go b/modules/git/tree_nogogit.go index d0ddb1d041..b8561dd352 100644 --- a/modules/git/tree_nogogit.go +++ b/modules/git/tree_nogogit.go @@ -27,12 +27,14 @@ func (t *Tree) ListEntries() (Entries, error) { } if t.repo != nil { - wr, rd, cancel, err := t.repo.CatFileBatch(t.repo.Ctx) + batch, cancel, err := t.repo.CatFileBatch(t.repo.Ctx) if err != nil { return nil, err } defer cancel() + wr := batch.Writer() + rd := batch.Reader() _, _ = wr.Write([]byte(t.ID.String() + "\n")) _, typ, sz, err := ReadBatchLine(rd) if err != nil { diff --git a/modules/gitrepo/cat_file.go b/modules/gitrepo/cat_file.go index c6ac74756f..0e5fc9951c 100644 --- a/modules/gitrepo/cat_file.go +++ b/modules/gitrepo/cat_file.go @@ -6,9 +6,9 @@ package gitrepo import ( "context" - "code.gitea.io/gitea/modules/git" + "code.gitea.io/gitea/modules/git/catfile" ) -func NewBatch(ctx context.Context, repo Repository) (*git.Batch, error) { - return git.NewBatch(ctx, repoPath(repo)) +func NewBatch(ctx context.Context, repo Repository) (catfile.Batch, error) { + return catfile.NewBatch(ctx, repoPath(repo)) } diff --git a/modules/indexer/code/bleve/bleve.go b/modules/indexer/code/bleve/bleve.go index 5f6a7f6082..a3727bd0cb 100644 --- a/modules/indexer/code/bleve/bleve.go +++ b/modules/indexer/code/bleve/bleve.go @@ -4,7 +4,6 @@ package bleve import ( - "bufio" "context" "fmt" "io" @@ -16,6 +15,7 @@ import ( "code.gitea.io/gitea/modules/analyze" "code.gitea.io/gitea/modules/charset" "code.gitea.io/gitea/modules/git" + "code.gitea.io/gitea/modules/git/catfile" "code.gitea.io/gitea/modules/git/gitcmd" "code.gitea.io/gitea/modules/gitrepo" "code.gitea.io/gitea/modules/indexer" @@ -151,7 +151,7 @@ func NewIndexer(indexDir string) *Indexer { } } -func (b *Indexer) addUpdate(ctx context.Context, batchWriter git.WriteCloserError, batchReader *bufio.Reader, commitSha string, +func (b *Indexer) addUpdate(ctx context.Context, catfileBatch catfile.Batch, commitSha string, update internal.FileUpdate, repo *repo_model.Repository, batch *inner_bleve.FlushingBatch, ) error { // Ignore vendored files in code search @@ -177,10 +177,11 @@ func (b *Indexer) addUpdate(ctx context.Context, batchWriter git.WriteCloserErro return b.addDelete(update.Filename, repo, batch) } - if _, err := batchWriter.Write([]byte(update.BlobSha + "\n")); err != nil { + if _, err := catfileBatch.Writer().Write([]byte(update.BlobSha + "\n")); err != nil { return err } + batchReader := catfileBatch.Reader() _, _, size, err = git.ReadBatchLine(batchReader) if err != nil { return err @@ -218,18 +219,18 @@ func (b *Indexer) addDelete(filename string, repo *repo_model.Repository, batch func (b *Indexer) Index(ctx context.Context, repo *repo_model.Repository, sha string, changes *internal.RepoChanges) error { batch := inner_bleve.NewFlushingBatch(b.inner.Indexer, maxBatchSize) if len(changes.Updates) > 0 { - gitBatch, err := gitrepo.NewBatch(ctx, repo) + catfileBatch, err := gitrepo.NewBatch(ctx, repo) if err != nil { return err } - defer gitBatch.Close() + defer catfileBatch.Close() for _, update := range changes.Updates { - if err := b.addUpdate(ctx, gitBatch.Writer, gitBatch.Reader, sha, update, repo, batch); err != nil { + if err := b.addUpdate(ctx, catfileBatch, sha, update, repo, batch); err != nil { return err } } - gitBatch.Close() + catfileBatch.Close() } for _, filename := range changes.RemovedFilenames { if err := b.addDelete(filename, repo, batch); err != nil { diff --git a/modules/indexer/code/elasticsearch/elasticsearch.go b/modules/indexer/code/elasticsearch/elasticsearch.go index a7027051d2..653df0bd11 100644 --- a/modules/indexer/code/elasticsearch/elasticsearch.go +++ b/modules/indexer/code/elasticsearch/elasticsearch.go @@ -4,7 +4,6 @@ package elasticsearch import ( - "bufio" "context" "fmt" "io" @@ -15,6 +14,7 @@ import ( "code.gitea.io/gitea/modules/analyze" "code.gitea.io/gitea/modules/charset" "code.gitea.io/gitea/modules/git" + "code.gitea.io/gitea/modules/git/catfile" "code.gitea.io/gitea/modules/git/gitcmd" "code.gitea.io/gitea/modules/gitrepo" "code.gitea.io/gitea/modules/indexer" @@ -139,7 +139,7 @@ const ( }` ) -func (b *Indexer) addUpdate(ctx context.Context, batchWriter git.WriteCloserError, batchReader *bufio.Reader, sha string, update internal.FileUpdate, repo *repo_model.Repository) ([]elastic.BulkableRequest, error) { +func (b *Indexer) addUpdate(ctx context.Context, batch catfile.Batch, sha string, update internal.FileUpdate, repo *repo_model.Repository) ([]elastic.BulkableRequest, error) { // Ignore vendored files in code search if setting.Indexer.ExcludeVendored && analyze.IsVendor(update.Filename) { return nil, nil @@ -162,10 +162,11 @@ func (b *Indexer) addUpdate(ctx context.Context, batchWriter git.WriteCloserErro return []elastic.BulkableRequest{b.addDelete(update.Filename, repo)}, nil } - if _, err := batchWriter.Write([]byte(update.BlobSha + "\n")); err != nil { + if _, err := batch.Writer().Write([]byte(update.BlobSha + "\n")); err != nil { return nil, err } + batchReader := batch.Reader() _, _, size, err = git.ReadBatchLine(batchReader) if err != nil { return nil, err @@ -217,7 +218,7 @@ func (b *Indexer) Index(ctx context.Context, repo *repo_model.Repository, sha st defer batch.Close() for _, update := range changes.Updates { - updateReqs, err := b.addUpdate(ctx, batch.Writer, batch.Reader, sha, update, repo) + updateReqs, err := b.addUpdate(ctx, batch, sha, update, repo) if err != nil { return err }