Optimize reading gist files content (#186)

This commit is contained in:
Thomas Miceli 2023-12-30 23:46:14 +01:00
parent f557bd45df
commit b3a856a05e
6 changed files with 130 additions and 14 deletions

View file

@ -311,24 +311,24 @@ func (gist *Gist) DeleteRepository() error {
} }
func (gist *Gist) Files(revision string, truncate bool) ([]*git.File, error) { func (gist *Gist) Files(revision string, truncate bool) ([]*git.File, error) {
var files []*git.File filesCat, err := git.CatFileBatch(gist.User.Username, gist.Uuid, revision, truncate)
filesStr, err := git.GetFilesOfRepository(gist.User.Username, gist.Uuid, revision)
if err != nil { if err != nil {
// if the revision or the file do not exist // if the revision or the file do not exist
if exiterr, ok := err.(*exec.ExitError); ok && exiterr.ExitCode() == 128 { if exiterr, ok := err.(*exec.ExitError); ok && exiterr.ExitCode() == 128 {
return nil, &git.RevisionNotFoundError{} return nil, &git.RevisionNotFoundError{}
} }
return nil, err return nil, err
} }
for _, fileStr := range filesStr { var files []*git.File
file, err := gist.File(revision, fileStr, truncate) for _, fileCat := range filesCat {
if err != nil { files = append(files, &git.File{
return nil, err Filename: fileCat.Name,
} Size: fileCat.Size,
files = append(files, file) HumanSize: humanize.IBytes(fileCat.Size),
Content: fileCat.Content,
Truncated: fileCat.Truncated,
})
} }
return files, err return files, err
} }

View file

@ -1,9 +1,11 @@
package git package git
import ( import (
"bufio"
"bytes" "bytes"
"context" "context"
"fmt" "fmt"
"io"
"os" "os"
"os/exec" "os/exec"
"path" "path"
@ -124,6 +126,120 @@ func GetFilesOfRepository(user string, gist string, revision string) ([]string,
return slice[:len(slice)-1], nil return slice[:len(slice)-1], nil
} }
type catFileBatch struct {
Name, Hash, Content string
Size uint64
Truncated bool
}
func CatFileBatch(user string, gist string, revision string, truncate bool) ([]*catFileBatch, error) {
repositoryPath := RepositoryPath(user, gist)
lsTreeCmd := exec.Command("git", "ls-tree", "-l", revision)
lsTreeCmd.Dir = repositoryPath
lsTreeOutput, err := lsTreeCmd.Output()
if err != nil {
return nil, err
}
fileMap := make([]*catFileBatch, 0)
lines := strings.Split(string(lsTreeOutput), "\n")
for _, line := range lines {
fields := strings.Fields(line)
if len(fields) < 4 {
continue // Skip lines that don't have enough fields
}
hash := fields[2]
size, err := strconv.ParseUint(fields[3], 10, 64)
if err != nil {
continue // Skip lines with invalid size field
}
name := strings.Join(fields[4:], " ") // File name may contain spaces
fileMap = append(fileMap, &catFileBatch{
Hash: hash,
Size: size,
Name: name,
})
}
catFileCmd := exec.Command("git", "cat-file", "--batch")
catFileCmd.Dir = repositoryPath
stdin, err := catFileCmd.StdinPipe()
if err != nil {
return nil, err
}
stdout, err := catFileCmd.StdoutPipe()
if err != nil {
return nil, err
}
if err = catFileCmd.Start(); err != nil {
return nil, err
}
reader := bufio.NewReader(stdout)
for _, file := range fileMap {
_, err = stdin.Write([]byte(file.Hash + "\n"))
if err != nil {
return nil, err
}
header, err := reader.ReadString('\n')
if err != nil {
return nil, err
}
parts := strings.Fields(header)
if len(parts) > 3 {
continue // Not a valid header, skip this entry
}
size, err := strconv.ParseUint(parts[2], 10, 64)
if err != nil {
return nil, err
}
sizeToRead := size
if truncate && sizeToRead > truncateLimit {
sizeToRead = truncateLimit
}
// Read exactly size bytes from header, or the max allowed if truncated
content := make([]byte, sizeToRead)
if _, err = io.ReadFull(reader, content); err != nil {
return nil, err
}
file.Content = string(content)
if truncate && size > truncateLimit {
// skip other bytes if truncated
if _, err = reader.Discard(int(size - truncateLimit)); err != nil {
return nil, err
}
file.Truncated = true
}
// Read the blank line following the content
if _, err := reader.ReadByte(); err != nil {
return nil, err
}
}
if err = stdin.Close(); err != nil {
return nil, err
}
if err = catFileCmd.Wait(); err != nil {
return nil, err
}
return fileMap, nil
}
func GetFileContent(user string, gist string, revision string, filename string, truncate bool) (string, bool, error) { func GetFileContent(user string, gist string, revision string, filename string, truncate bool) (string, bool, error) {
repositoryPath := RepositoryPath(user, gist) repositoryPath := RepositoryPath(user, gist)

View file

@ -39,7 +39,7 @@ func HighlightFile(file *git.File) (RenderedFile, error) {
formatter := html.New(html.WithClasses(true), html.PreventSurroundingPre(true)) formatter := html.New(html.WithClasses(true), html.PreventSurroundingPre(true))
iterator, err := lexer.Tokenise(nil, file.Content) iterator, err := lexer.Tokenise(nil, file.Content+"\n")
if err != nil { if err != nil {
return rendered, err return rendered, err
} }

View file

@ -704,7 +704,7 @@ func downloadZip(ctx echo.Context) error {
gist := getData(ctx, "gist").(*db.Gist) gist := getData(ctx, "gist").(*db.Gist)
revision := ctx.Param("revision") revision := ctx.Param("revision")
files, err := gist.Files(revision, true) files, err := gist.Files(revision, false)
if err != nil { if err != nil {
return errorRes(500, "Error fetching files from repository", err) return errorRes(500, "Error fetching files from repository", err)
} }

2
public/embed.scss vendored
View file

@ -107,6 +107,6 @@ dl.dl-config dd {
@apply overflow-auto whitespace-pre; @apply overflow-auto whitespace-pre;
} }
.chroma.preview.markdown code { .chroma.preview.markdown pre code {
@apply p-4; @apply p-4;
} }

2
public/style.css vendored
View file

@ -167,7 +167,7 @@ dl.dl-config dd {
@apply overflow-auto whitespace-pre !important; @apply overflow-auto whitespace-pre !important;
} }
.chroma.preview.markdown code { .chroma.preview.markdown pre code {
@apply p-4 !important; @apply p-4 !important;
} }