From 527be16183a43ffa3efff2ade1e2fa2a6157d5f6 Mon Sep 17 00:00:00 2001 From: Thomas Miceli Date: Sat, 18 Mar 2023 23:18:20 +0100 Subject: [PATCH] Better log parsing --- internal/git/commands.go | 16 ++--- internal/git/output_parser.go | 112 +++++++++++++++++++++++++++++++++ internal/models/gist.go | 41 ++++-------- internal/web/gist.go | 51 ++------------- templates/pages/gist.html | 16 ++--- templates/pages/revisions.html | 25 ++++---- 6 files changed, 158 insertions(+), 103 deletions(-) diff --git a/internal/git/commands.go b/internal/git/commands.go index 77610d2..a5128d2 100644 --- a/internal/git/commands.go +++ b/internal/git/commands.go @@ -121,7 +121,7 @@ func GetFileContent(user string, gist string, revision string, filename string, return truncateCommandOutput(stdout, maxBytes) } -func GetLog(user string, gist string, skip string) (string, error) { +func GetLog(user string, gist string, skip string) ([]*Commit, error) { repositoryPath := RepositoryPath(user, gist) cmd := exec.Command( @@ -130,20 +130,22 @@ func GetLog(user string, gist string, skip string) (string, error) { "log", "-n", "11", - "--no-prefix", "--no-color", "-p", "--skip", skip, - "--format=format:%n=commit %H:%aN:%at", + "--format=format:c %H%na %aN%nt %at", "--shortstat", - "--ignore-missing", // avoid errors if a wrong hash is given "HEAD", ) cmd.Dir = repositoryPath + stdout, _ := cmd.StdoutPipe() + err := cmd.Start() + if err != nil { + return nil, err + } - stdout, err := cmd.Output() - return string(stdout), err + return parseLog(stdout), nil } func CloneTmp(user string, gist string, gistTmpId string) error { @@ -213,7 +215,7 @@ func AddAll(gistTmpId string) error { return cmd.Run() } -func Commit(gistTmpId string) error { +func CommitRepository(gistTmpId string) error { cmd := exec.Command("git", "commit", "--allow-empty", "-m", `"Opengist commit"`) tmpPath := TmpRepositoryPath(gistTmpId) cmd.Dir = tmpPath diff --git a/internal/git/output_parser.go b/internal/git/output_parser.go index 3ba0504..8e95779 100644 --- a/internal/git/output_parser.go +++ b/internal/git/output_parser.go @@ -1,10 +1,29 @@ package git import ( + "bufio" "bytes" "io" + "regexp" ) +type File struct { + Filename string `validate:"excludes=\x2f,excludes=\x5c,max=50"` + OldFilename string `validate:"excludes=\x2f,excludes=\x5c,max=50"` + Content string `validate:"required"` + Truncated bool + IsCreated bool + IsDeleted bool +} + +type Commit struct { + Hash string + Author string + Timestamp string + Changed string + Files []File +} + func truncateCommandOutput(out io.Reader, maxBytes int64) (string, bool, error) { var buf []byte var err error @@ -31,3 +50,96 @@ func truncateCommandOutput(out io.Reader, maxBytes int64) (string, bool, error) return string(buf), truncated, nil } + +func parseLog(out io.Reader) []*Commit { + scanner := bufio.NewScanner(out) + + var commits []*Commit + var currentCommit *Commit + var currentFile *File + var isContent bool + + for scanner.Scan() { + // new commit found + currentFile = nil + currentCommit = &Commit{Hash: string(scanner.Bytes()[2:]), Files: []File{}} + + scanner.Scan() + currentCommit.Author = string(scanner.Bytes()[2:]) + + scanner.Scan() + currentCommit.Timestamp = string(scanner.Bytes()[2:]) + + scanner.Scan() + changed := scanner.Bytes()[1:] + changed = bytes.ReplaceAll(changed, []byte("(+)"), []byte("")) + changed = bytes.ReplaceAll(changed, []byte("(-)"), []byte("")) + currentCommit.Changed = string(changed) + + // twice because --shortstat adds a new line + scanner.Scan() + scanner.Scan() + // commit header parsed + + // files changes inside the commit + for { + line := scanner.Bytes() + + // end of content of file + if len(line) == 0 { + isContent = false + if currentFile != nil { + currentCommit.Files = append(currentCommit.Files, *currentFile) + } + break + } + + // new file found + if bytes.HasPrefix(line, []byte("diff --git")) { + // current file is finished, we can add it to the commit + if currentFile != nil { + currentCommit.Files = append(currentCommit.Files, *currentFile) + } + + // create a new file + isContent = false + currentFile = &File{} + filenameRegex := regexp.MustCompile(`^diff --git a/(.+) b/(.+)$`) + matches := filenameRegex.FindStringSubmatch(string(line)) + if len(matches) == 3 { + currentFile.Filename = matches[2] + if matches[1] != matches[2] { + currentFile.OldFilename = matches[1] + } + } + scanner.Scan() + continue + } + + if bytes.HasPrefix(line, []byte("new")) { + currentFile.IsCreated = true + } + + if bytes.HasPrefix(line, []byte("deleted")) { + currentFile.IsDeleted = true + } + + // file content found + if line[0] == '@' { + isContent = true + } + + if isContent { + currentFile.Content += string(line) + "\n" + } + + scanner.Scan() + } + + if currentCommit != nil { + commits = append(commits, currentCommit) + } + } + + return commits +} diff --git a/internal/models/gist.go b/internal/models/gist.go index 3c1c0d6..6f54eb0 100644 --- a/internal/models/gist.go +++ b/internal/models/gist.go @@ -28,21 +28,6 @@ type Gist struct { ForkedID uint } -type File struct { - Filename string `validate:"excludes=\x2f,excludes=\x5c,max=50"` - OldFilename string `validate:"excludes=\x2f,excludes=\x5c,max=50"` - Content string `validate:"required"` - Truncated bool -} - -type Commit struct { - Hash string - Author string - Timestamp string - Changed string - Files []File -} - func (gist *Gist) BeforeDelete(tx *gorm.DB) error { // Decrement fork counter if the gist was forked err := tx.Model(&Gist{}). @@ -195,8 +180,8 @@ func (gist *Gist) DeleteRepository() error { return git.DeleteRepository(gist.User.Username, gist.Uuid) } -func (gist *Gist) Files(revision string) ([]*File, error) { - var files []*File +func (gist *Gist) Files(revision string) ([]*git.File, error) { + var files []*git.File filesStr, err := git.GetFilesOfRepository(gist.User.Username, gist.Uuid, revision) if err != nil { // if the revision or the file do not exist @@ -218,7 +203,7 @@ func (gist *Gist) Files(revision string) ([]*File, error) { return files, err } -func (gist *Gist) File(revision string, filename string, truncate bool) (*File, error) { +func (gist *Gist) File(revision string, filename string, truncate bool) (*git.File, error) { content, truncated, err := git.GetFileContent(gist.User.Username, gist.Uuid, revision, filename, truncate) // if the revision or the file do not exist @@ -226,24 +211,22 @@ func (gist *Gist) File(revision string, filename string, truncate bool) (*File, return nil, nil } - return &File{ + return &git.File{ Filename: filename, Content: content, Truncated: truncated, }, err } -func (gist *Gist) Log(skip string) error { - _, err := git.GetLog(gist.User.Username, gist.Uuid, skip) - - return err +func (gist *Gist) Log(skip string) ([]*git.Commit, error) { + return git.GetLog(gist.User.Username, gist.Uuid, skip) } func (gist *Gist) NbCommits() (string, error) { return git.GetNumberOfCommitsOfRepository(gist.User.Username, gist.Uuid) } -func (gist *Gist) AddAndCommitFiles(files *[]File) error { +func (gist *Gist) AddAndCommitFiles(files *[]git.File) error { if err := git.CloneTmp(gist.User.Username, gist.Uuid, gist.Uuid); err != nil { return err } @@ -258,7 +241,7 @@ func (gist *Gist) AddAndCommitFiles(files *[]File) error { return err } - if err := git.Commit(gist.Uuid); err != nil { + if err := git.CommitRepository(gist.Uuid); err != nil { return err } @@ -280,10 +263,10 @@ func (gist *Gist) RPC(service string) ([]byte, error) { // -- DTO -- // type GistDTO struct { - Title string `validate:"max=50" form:"title"` - Description string `validate:"max=150" form:"description"` - Private bool `form:"private"` - Files []File `validate:"min=1,dive"` + Title string `validate:"max=50" form:"title"` + Description string `validate:"max=150" form:"description"` + Private bool `form:"private"` + Files []git.File `validate:"min=1,dive"` } func (dto *GistDTO) ToGist() *Gist { diff --git a/internal/web/gist.go b/internal/web/gist.go index 2384756..e2e33e0 100644 --- a/internal/web/gist.go +++ b/internal/web/gist.go @@ -162,50 +162,9 @@ func revisions(ctx echo.Context) error { pageInt := getPage(ctx) - nbCommits := getData(ctx, "nbCommits") - commits := make([]*models.Commit, 0) - if nbCommits != "0" { - gitlogStr, err := git.GetLog(userName, gistName, strconv.Itoa((pageInt-1)*10)) - if err != nil { - return errorRes(500, "Error fetching commits log", err) - } - - gitlog := strings.Split(gitlogStr, "\n=commit ") - for _, commitStr := range gitlog[1:] { - logContent := strings.SplitN(commitStr, "\n", 3) - - header := strings.Split(logContent[0], ":") - commitStruct := models.Commit{ - Hash: header[0], - Author: header[1], - Timestamp: header[2], - Files: make([]models.File, 0), - } - - if len(logContent) > 2 { - changed := strings.ReplaceAll(logContent[1], "(+)", "") - changed = strings.ReplaceAll(changed, "(-)", "") - commitStruct.Changed = changed - } - - files := strings.Split(logContent[len(logContent)-1], "diff --git ") - if len(files) > 1 { - for _, fileStr := range files { - content := strings.SplitN(fileStr, "\n@@", 2) - if len(content) > 1 { - header := strings.Split(content[0], "\n") - commitStruct.Files = append(commitStruct.Files, models.File{Content: "@@" + content[1], Filename: header[len(header)-1][4:], OldFilename: header[len(header)-2][4:]}) - } else { - // in case there is no content but a file renamed - header := strings.Split(content[0], "\n") - if len(header) > 3 { - commitStruct.Files = append(commitStruct.Files, models.File{Content: "", Filename: header[3][10:], OldFilename: header[2][12:]}) - } - } - } - } - commits = append(commits, &commitStruct) - } + commits, err := gist.Log(strconv.Itoa((pageInt - 1) * 10)) + if err != nil { + return errorRes(500, "Error fetching commits log", err) } if err := paginate(ctx, commits, pageInt, 10, "commits", userName+"/"+gistName+"/revisions", 2); err != nil { @@ -249,7 +208,7 @@ func processCreate(ctx echo.Context) error { return errorRes(400, "Cannot bind data", err) } - dto.Files = make([]models.File, 0) + dto.Files = make([]git.File, 0) for i := 0; i < len(ctx.Request().PostForm["content"]); i++ { name := ctx.Request().PostForm["name"][i] content := ctx.Request().PostForm["content"][i] @@ -263,7 +222,7 @@ func processCreate(ctx echo.Context) error { return errorRes(400, "Invalid character unescaped", err) } - dto.Files = append(dto.Files, models.File{ + dto.Files = append(dto.Files, git.File{ Filename: name, Content: escapedValue, }) diff --git a/templates/pages/gist.html b/templates/pages/gist.html index 4a5c22f..3169896 100644 --- a/templates/pages/gist.html +++ b/templates/pages/gist.html @@ -26,13 +26,15 @@
{{ $file.Content }}
{{ else }} {{ $fileslug := slug $file.Filename }} - - - {{ $ii := "1" }} - {{ $i := toInt $ii }} - {{ range $line := lines $file.Content }}{{ $i = inc $i }}{{ end }} - -
{{$i}}{{ $line }}
+ {{ if ne $file.Content "" }} + + + {{ $ii := "1" }} + {{ $i := toInt $ii }} + {{ range $line := lines $file.Content }}{{ $i = inc $i }}{{ end }} + +
{{$i}}{{ $line }}
+ {{ end }} {{ end }} diff --git a/templates/pages/revisions.html b/templates/pages/revisions.html index 7c29062..d1eef7c 100644 --- a/templates/pages/revisions.html +++ b/templates/pages/revisions.html @@ -27,24 +27,22 @@ - {{ if eq $file.Filename $file.OldFilename }} - {{ $file.Filename }} + {{ if $file.IsCreated }} + {{ $file.Filename }}(file created) + {{ else if $file.IsDeleted }} + {{ $file.Filename }} (file deleted) + {{ else if ne $file.OldFilename "" }} + {{ $file.OldFilename }} renamed to {{ $file.Filename }} {{ else }} - {{ if eq $file.OldFilename "/dev/null" }} - {{ $file.Filename }}(file created) - {{ else if eq $file.Filename "/dev/null" }} - {{ $file.OldFilename }} (file deleted) - {{ else }} - {{ $file.OldFilename }} renamed to {{ $file.Filename }} - {{ end }} + {{ $file.Filename }} {{ end }}

- {{ if eq $file.Content "" }} -

- File renamed without changes. -

+ {{ if and (eq $file.Content "") (ne $file.OldFilename "") }} +

File renamed without changes

+ {{ else if eq $file.Content "" }} +

Empty file

{{ else }} @@ -52,7 +50,6 @@ {{ $right := 0 }} {{ range $line := split $file.Content "\n" }} {{ if ne $line "" }}{{ if ne (index $line 0) 92 }} - {{ if eq (index $line 0) 64 }} {{ $left = toInt (index (splitGit (index (split $line "-") 1)) 0) }} {{ $right = toInt (index (splitGit (index (split $line "+") 1)) 0) }}