Optimize reading gist files content (#186)

internal/db/gist.go

@@ -311,24 +311,24 @@ func (gist *Gist) DeleteRepository() error {
 }
 
 func (gist *Gist) Files(revision string, truncate bool) ([]*git.File, error) {
-	var files []*git.File
-	filesStr, err := git.GetFilesOfRepository(gist.User.Username, gist.Uuid, revision)
+	filesCat, err := git.CatFileBatch(gist.User.Username, gist.Uuid, revision, truncate)
 	if err != nil {
 		// if the revision or the file do not exist
-
 		if exiterr, ok := err.(*exec.ExitError); ok && exiterr.ExitCode() == 128 {
 			return nil, &git.RevisionNotFoundError{}
 		}
-
 		return nil, err
 	}
 
-	for _, fileStr := range filesStr {
-		file, err := gist.File(revision, fileStr, truncate)
-		if err != nil {
-			return nil, err
-		}
-		files = append(files, file)
+	var files []*git.File
+	for _, fileCat := range filesCat {
+		files = append(files, &git.File{
+			Filename:  fileCat.Name,
+			Size:      fileCat.Size,
+			HumanSize: humanize.IBytes(fileCat.Size),
+			Content:   fileCat.Content,
+			Truncated: fileCat.Truncated,
+		})
 	}
 	return files, err
 }

internal/git/commands.go

@@ -1,9 +1,11 @@
 package git
 
 import (
+	"bufio"
 	"bytes"
 	"context"
 	"fmt"
+	"io"
 	"os"
 	"os/exec"
 	"path"
@@ -124,6 +126,120 @@ func GetFilesOfRepository(user string, gist string, revision string) ([]string,
 	return slice[:len(slice)-1], nil
 }
 
+type catFileBatch struct {
+	Name, Hash, Content string
+	Size                uint64
+	Truncated           bool
+}
+
+func CatFileBatch(user string, gist string, revision string, truncate bool) ([]*catFileBatch, error) {
+	repositoryPath := RepositoryPath(user, gist)
+
+	lsTreeCmd := exec.Command("git", "ls-tree", "-l", revision)
+	lsTreeCmd.Dir = repositoryPath
+	lsTreeOutput, err := lsTreeCmd.Output()
+	if err != nil {
+		return nil, err
+	}
+
+	fileMap := make([]*catFileBatch, 0)
+
+	lines := strings.Split(string(lsTreeOutput), "\n")
+	for _, line := range lines {
+		fields := strings.Fields(line)
+		if len(fields) < 4 {
+			continue // Skip lines that don't have enough fields
+		}
+
+		hash := fields[2]
+		size, err := strconv.ParseUint(fields[3], 10, 64)
+		if err != nil {
+			continue // Skip lines with invalid size field
+		}
+		name := strings.Join(fields[4:], " ") // File name may contain spaces
+
+		fileMap = append(fileMap, &catFileBatch{
+			Hash: hash,
+			Size: size,
+			Name: name,
+		})
+	}
+
+	catFileCmd := exec.Command("git", "cat-file", "--batch")
+	catFileCmd.Dir = repositoryPath
+	stdin, err := catFileCmd.StdinPipe()
+	if err != nil {
+		return nil, err
+	}
+	stdout, err := catFileCmd.StdoutPipe()
+	if err != nil {
+		return nil, err
+	}
+	if err = catFileCmd.Start(); err != nil {
+		return nil, err
+	}
+
+	reader := bufio.NewReader(stdout)
+
+	for _, file := range fileMap {
+		_, err = stdin.Write([]byte(file.Hash + "\n"))
+		if err != nil {
+			return nil, err
+		}
+
+		header, err := reader.ReadString('\n')
+		if err != nil {
+			return nil, err
+		}
+
+		parts := strings.Fields(header)
+		if len(parts) > 3 {
+			continue // Not a valid header, skip this entry
+		}
+
+		size, err := strconv.ParseUint(parts[2], 10, 64)
+		if err != nil {
+			return nil, err
+		}
+
+		sizeToRead := size
+		if truncate && sizeToRead > truncateLimit {
+			sizeToRead = truncateLimit
+		}
+
+		// Read exactly size bytes from header, or the max allowed if truncated
+		content := make([]byte, sizeToRead)
+		if _, err = io.ReadFull(reader, content); err != nil {
+			return nil, err
+		}
+
+		file.Content = string(content)
+
+		if truncate && size > truncateLimit {
+			// skip other bytes if truncated
+			if _, err = reader.Discard(int(size - truncateLimit)); err != nil {
+				return nil, err
+			}
+			file.Truncated = true
+		}
+
+		// Read the blank line following the content
+		if _, err := reader.ReadByte(); err != nil {
+			return nil, err
+		}
+	}
+
+	if err = stdin.Close(); err != nil {
+		return nil, err
+	}
+
+	if err = catFileCmd.Wait(); err != nil {
+		return nil, err
+	}
+
+	return fileMap, nil
+}
+
 func GetFileContent(user string, gist string, revision string, filename string, truncate bool) (string, bool, error) {
 	repositoryPath := RepositoryPath(user, gist)
 

internal/render/highlight.go

@@ -39,7 +39,7 @@ func HighlightFile(file *git.File) (RenderedFile, error) {
 
 	formatter := html.New(html.WithClasses(true), html.PreventSurroundingPre(true))
 
-	iterator, err := lexer.Tokenise(nil, file.Content)
+	iterator, err := lexer.Tokenise(nil, file.Content+"\n")
 	if err != nil {
 		return rendered, err
 	}

internal/web/gist.go

@@ -704,7 +704,7 @@ func downloadZip(ctx echo.Context) error {
 	gist := getData(ctx, "gist").(*db.Gist)
 	revision := ctx.Param("revision")
 
-	files, err := gist.Files(revision, true)
+	files, err := gist.Files(revision, false)
 	if err != nil {
 		return errorRes(500, "Error fetching files from repository", err)
 	}

public/embed.scss

@@ -107,6 +107,6 @@ dl.dl-config dd {
     @apply overflow-auto whitespace-pre;
 }
 
-.chroma.preview.markdown code {
+.chroma.preview.markdown pre code {
     @apply p-4;
 }

public/style.css

@@ -167,7 +167,7 @@ dl.dl-config dd {
     @apply overflow-auto whitespace-pre !important;
 }
 
-.chroma.preview.markdown code {
+.chroma.preview.markdown pre code {
     @apply p-4 !important;
 }