Rework git log parsing and truncating (#260)

This commit is contained in:
Thomas Miceli 2024-04-27 01:49:53 +02:00 committed by GitHub
parent 6a8759e21e
commit 785d89d6ab
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
4 changed files with 268 additions and 109 deletions

View file

@ -24,6 +24,8 @@ var (
) )
const truncateLimit = 2 << 18 const truncateLimit = 2 << 18
const diffSize = 2 << 12
const maxFilesPerDiffCommit = 10
type RevisionNotFoundError struct{} type RevisionNotFoundError struct{}
@ -313,7 +315,7 @@ func GetLog(user string, gist string, skip int) ([]*Commit, error) {
} }
}(cmd) }(cmd)
return parseLog(stdout, truncateLimit), err return parseLog(stdout, maxFilesPerDiffCommit, diffSize)
} }
func CloneTmp(user string, gist string, gistTmpId string, email string, remove bool) error { func CloneTmp(user string, gist string, gistTmpId string, email string, remove bool) error {

View file

@ -125,7 +125,7 @@ like Opengist actually`,
require.Contains(t, commits[0].Files, File{ require.Contains(t, commits[0].Files, File{
Filename: "my_other_file.txt", Filename: "my_other_file.txt",
OldFilename: "", OldFilename: "my_other_file.txt",
Content: `@@ -1,2 +1,2 @@ Content: `@@ -1,2 +1,2 @@
I really I really
-hate Opengist -hate Opengist

View file

@ -6,7 +6,6 @@ import (
"encoding/csv" "encoding/csv"
"fmt" "fmt"
"io" "io"
"regexp"
"strings" "strings"
) )
@ -63,129 +62,287 @@ func truncateCommandOutput(out io.Reader, maxBytes int64) (string, bool, error)
return string(buf), truncated, nil return string(buf), truncated, nil
} }
func parseLog(out io.Reader, maxBytes int) []*Commit { // inspired from https://github.com/go-gitea/gitea/blob/main/services/gitdiff/gitdiff.go
scanner := bufio.NewScanner(out) func parseLog(out io.Reader, maxFiles int, maxBytes int) ([]*Commit, error) {
var commits []*Commit var commits []*Commit
var currentCommit *Commit var currentCommit *Commit
var currentFile *File var currentFile *File
var isContent bool var headerParsed = false
var bytesRead = 0 var skipped = false
scanNext := true var line string
var err error
input := bufio.NewReaderSize(out, maxBytes)
// Loop Commits
loopLog:
for { for {
if scanNext && !scanner.Scan() { // If a commit was skipped, do not read a new line
break if !skipped {
line, err = input.ReadString('\n')
if err != nil {
if err == io.EOF {
break loopLog
}
return commits, err
} }
scanNext = true
// new commit found
currentFile = nil
currentCommit = &Commit{Hash: string(scanner.Bytes()[2:]), Files: []File{}}
scanner.Scan()
currentCommit.AuthorName = string(scanner.Bytes()[2:])
scanner.Scan()
currentCommit.AuthorEmail = string(scanner.Bytes()[2:])
scanner.Scan()
currentCommit.Timestamp = string(scanner.Bytes()[2:])
scanner.Scan()
if len(scanner.Bytes()) == 0 {
commits = append(commits, currentCommit)
break
} }
// if there is no shortstat, it means that the commit is empty, we add it and move onto the next one // Remove trailing newline characters
if scanner.Bytes()[0] != ' ' { if len(line) > 0 && (line[len(line)-1] == '\n' || line[len(line)-1] == '\r') {
commits = append(commits, currentCommit) line = line[:len(line)-1]
}
// avoid scanning the next line, as we already did it // Attempt to parse commit header (hash, author, mail, timestamp) or a diff
scanNext = false switch line[0] {
// Commit hash
case 'c':
if headerParsed {
commits = append(commits, currentCommit)
}
skipped = false
currentCommit = &Commit{Hash: line[2:], Files: []File{}}
continue continue
}
changed := scanner.Bytes()[1:] // Author name
case 'a':
headerParsed = true
currentCommit.AuthorName = line[2:]
continue
// Author email
case 'm':
currentCommit.AuthorEmail = line[2:]
continue
// Commit timestamp
case 't':
currentCommit.Timestamp = line[2:]
continue
// Commit shortstat
case ' ':
changed := []byte(line)[1:]
changed = bytes.ReplaceAll(changed, []byte("(+)"), []byte("")) changed = bytes.ReplaceAll(changed, []byte("(+)"), []byte(""))
changed = bytes.ReplaceAll(changed, []byte("(-)"), []byte("")) changed = bytes.ReplaceAll(changed, []byte("(-)"), []byte(""))
currentCommit.Changed = string(changed) currentCommit.Changed = string(changed)
// twice because --shortstat adds a new line // shortstat is followed by an empty line
scanner.Scan() line, err = input.ReadString('\n')
scanner.Scan() if err != nil {
// commit header parsed if err == io.EOF {
break loopLog
}
return commits, err
}
continue
// files changes inside the commit // Commit diff
default:
// Loop files in diff
loopCommit:
for { for {
line := scanner.Bytes() // If we have reached the maximum number of files to show for a single commit, skip to the next commit
if len(currentCommit.Files) >= maxFiles {
// end of content of file line, err = skipToNextCommit(input)
if len(line) == 0 { if err != nil {
isContent = false if err == io.EOF {
if currentFile != nil { break loopLog
currentCommit.Files = append(currentCommit.Files, *currentFile)
} }
break return commits, err
} }
// new file found // Skip to the next commit
if bytes.HasPrefix(line, []byte("diff --git")) { headerParsed = false
// current file is finished, we can add it to the commit skipped = true
if currentFile != nil { break loopCommit
currentCommit.Files = append(currentCommit.Files, *currentFile)
} }
// create a new file // Else create a new file and parse it
isContent = false
bytesRead = 0
currentFile = &File{} currentFile = &File{}
filenameRegex := regexp.MustCompile(`^diff --git a/(.+) b/(.+)$`) parseRename := true
matches := filenameRegex.FindStringSubmatch(string(line))
if len(matches) == 3 { loopFileDiff:
currentFile.Filename = matches[2] for {
if matches[1] != matches[2] { line, err = input.ReadString('\n')
currentFile.OldFilename = matches[1] if err != nil {
if err != io.EOF {
return commits, err
}
headerParsed = false
break loopCommit
}
// If the line is a newline character, the commit is finished
if line == "\n" {
currentCommit.Files = append(currentCommit.Files, *currentFile)
headerParsed = false
break loopCommit
}
// Attempt to parse the file header
switch {
case strings.HasPrefix(line, "diff --git"):
currentCommit.Files = append(currentCommit.Files, *currentFile)
headerParsed = false
break loopFileDiff
case strings.HasPrefix(line, "old mode"):
case strings.HasPrefix(line, "new mode"):
case strings.HasPrefix(line, "index"):
case strings.HasPrefix(line, "similarity index"):
case strings.HasPrefix(line, "dissimilarity index"):
continue
case strings.HasPrefix(line, "rename from "):
currentFile.OldFilename = line[12 : len(line)-1]
case strings.HasPrefix(line, "rename to "):
currentFile.Filename = line[10 : len(line)-1]
parseRename = false
case strings.HasPrefix(line, "copy from "):
currentFile.OldFilename = line[10 : len(line)-1]
case strings.HasPrefix(line, "copy to "):
currentFile.Filename = line[8 : len(line)-1]
parseRename = false
case strings.HasPrefix(line, "new file"):
currentFile.IsCreated = true
case strings.HasPrefix(line, "deleted file"):
currentFile.IsDeleted = true
case strings.HasPrefix(line, "--- "):
name := line[4 : len(line)-1]
if parseRename && currentFile.IsDeleted {
currentFile.Filename = name[2:]
} else if parseRename && strings.HasPrefix(name, "a/") {
currentFile.OldFilename = name[2:]
}
case strings.HasPrefix(line, "+++ "):
name := line[4 : len(line)-1]
if parseRename && strings.HasPrefix(name, "b/") {
currentFile.Filename = name[2:]
}
// Header is finally parsed, now we can parse the file diff content
lineBytes, isFragment, err := parseDiffContent(currentFile, maxBytes, input)
if err != nil {
if err != io.EOF {
return commits, err
}
// EOF reached, commit is finished
currentCommit.Files = append(currentCommit.Files, *currentFile)
headerParsed = false
break loopCommit
}
currentCommit.Files = append(currentCommit.Files, *currentFile)
if string(lineBytes) == "" {
headerParsed = false
break loopCommit
}
for isFragment {
_, isFragment, err = input.ReadLine()
if err != nil {
return commits, fmt.Errorf("unable to ReadLine: %w", err)
} }
} }
scanner.Scan()
break loopFileDiff
}
}
}
}
commits = append(commits, currentCommit)
}
return commits, nil
}
func parseDiffContent(currentFile *File, maxBytes int, input *bufio.Reader) (lineBytes []byte, isFragment bool, err error) {
sb := &strings.Builder{}
var currFileLineCount int
for {
for isFragment {
currentFile.Truncated = true
// Read the next line
_, isFragment, err = input.ReadLine()
if err != nil {
return nil, false, err
}
}
sb.Reset()
// Read the next line
lineBytes, isFragment, err = input.ReadLine()
if err != nil {
if err == io.EOF {
return lineBytes, isFragment, err
}
return nil, false, err
}
// End of file
if len(lineBytes) == 0 {
return lineBytes, false, err
}
if lineBytes[0] == 'd' {
return lineBytes, false, err
}
if currFileLineCount >= maxBytes {
currentFile.Truncated = true
continue continue
} }
if bytes.HasPrefix(line, []byte("new")) { line := string(lineBytes)
currentFile.IsCreated = true if isFragment {
}
if bytes.HasPrefix(line, []byte("deleted")) {
currentFile.IsDeleted = true
}
// file content found
if line[0] == '@' {
isContent = true
}
if isContent {
currentFile.Content += string(line) + "\n"
bytesRead += len(line)
if bytesRead > maxBytes {
currentFile.Truncated = true currentFile.Truncated = true
currentFile.Content = "" for isFragment {
isContent = false lineBytes, isFragment, err = input.ReadLine()
if err != nil {
return lineBytes, isFragment, fmt.Errorf("unable to ReadLine: %w", err)
}
} }
} }
scanner.Scan() if len(line) > maxBytes {
currentFile.Truncated = true
line = line[:maxBytes]
}
currentFile.Content += line + "\n"
}
} }
commits = append(commits, currentCommit) func skipToNextCommit(input *bufio.Reader) (line string, err error) {
// need to skip until the next cmdDiffHead
var isFragment, wasFragment bool
var lineBytes []byte
for {
lineBytes, isFragment, err = input.ReadLine()
if err != nil {
return "", err
} }
if wasFragment {
return commits wasFragment = isFragment
continue
}
if bytes.HasPrefix(lineBytes, []byte("c")) {
break
}
wasFragment = isFragment
}
line = string(lineBytes)
if isFragment {
var tail string
tail, err = input.ReadString('\n')
if err != nil {
return "", err
}
line += tail
}
return line, err
} }
func ParseCsv(file *File) (*CsvFile, error) { func ParseCsv(file *File) (*CsvFile, error) {

View file

@ -35,7 +35,7 @@
<span class="flex text-sm ml-2 text-slate-700 dark:text-slate-300">{{ $file.Filename }}<span class="italic text-gray-600 dark:text-gray-400 ml-1">({{ $.locale.Tr "gist.revision.file-created" }})</span></span> <span class="flex text-sm ml-2 text-slate-700 dark:text-slate-300">{{ $file.Filename }}<span class="italic text-gray-600 dark:text-gray-400 ml-1">({{ $.locale.Tr "gist.revision.file-created" }})</span></span>
{{ else if $file.IsDeleted }} {{ else if $file.IsDeleted }}
<span class="flex text-sm ml-2 text-slate-700 dark:text-slate-300">{{ $file.Filename }} <span class="italic text-gray-600 dark:text-gray-400 ml-1">({{ $.locale.Tr "gist.revision.file-deleted" }})</span></span> <span class="flex text-sm ml-2 text-slate-700 dark:text-slate-300">{{ $file.Filename }} <span class="italic text-gray-600 dark:text-gray-400 ml-1">({{ $.locale.Tr "gist.revision.file-deleted" }})</span></span>
{{ else if ne $file.OldFilename "" }} {{ else if ne $file.OldFilename $file.Filename }}
<span class="flex text-sm ml-2 text-slate-700 dark:text-slate-300">{{ $file.OldFilename }} <span class="italic text-gray-600 dark:text-gray-400 mx-1">{{ $.locale.Tr "gist.revision.file-renamed" }}</span> {{ $file.Filename }}</span> <span class="flex text-sm ml-2 text-slate-700 dark:text-slate-300">{{ $file.OldFilename }} <span class="italic text-gray-600 dark:text-gray-400 mx-1">{{ $.locale.Tr "gist.revision.file-renamed" }}</span> {{ $file.Filename }}</span>
{{ else }} {{ else }}
<span class="flex text-sm ml-2 text-slate-700 dark:text-slate-300">{{ $file.Filename }}</span> <span class="flex text-sm ml-2 text-slate-700 dark:text-slate-300">{{ $file.Filename }}</span>