From 22668be9234cb766633915e051cd0723534b8253 Mon Sep 17 00:00:00 2001 From: Thomas Miceli Date: Sat, 18 Mar 2023 16:18:24 +0100 Subject: [PATCH] Added gist methods related to git + Truncate command output --- internal/git/commands.go | 26 +++++--- internal/git/output_parser.go | 39 ++++++++++++ internal/models/gist.go | 97 ++++++++++++++++++++++++++-- internal/web/admin.go | 2 +- internal/web/gist.go | 117 ++++++++++------------------------ internal/web/git-http.go | 4 +- templates/pages/edit.html | 6 +- templates/pages/gist.html | 23 ++++--- 8 files changed, 202 insertions(+), 112 deletions(-) create mode 100644 internal/git/output_parser.go diff --git a/internal/git/commands.go b/internal/git/commands.go index 3fd0e1e..77610d2 100644 --- a/internal/git/commands.go +++ b/internal/git/commands.go @@ -76,13 +76,13 @@ func GetNumberOfCommitsOfRepository(user string, gist string) (string, error) { return strings.TrimSuffix(string(stdout), "\n"), err } -func GetFilesOfRepository(user string, gist string, commit string) ([]string, error) { +func GetFilesOfRepository(user string, gist string, revision string) ([]string, error) { repositoryPath := RepositoryPath(user, gist) cmd := exec.Command( "git", "ls-tree", - commit, + revision, "--name-only", ) cmd.Dir = repositoryPath @@ -96,19 +96,29 @@ func GetFilesOfRepository(user string, gist string, commit string) ([]string, er return slice[:len(slice)-1], nil } -func GetFileContent(user string, gist string, commit string, filename string) (string, error) { +func GetFileContent(user string, gist string, revision string, filename string, truncate bool) (string, bool, error) { repositoryPath := RepositoryPath(user, gist) + var maxBytes int64 = -1 + if truncate { + maxBytes = 2 << 18 + } + cmd := exec.Command( "git", "--no-pager", "show", - commit+":"+filename, + revision+":"+filename, ) cmd.Dir = repositoryPath - stdout, err := cmd.Output() - return string(stdout), err + stdout, _ := cmd.StdoutPipe() + err := cmd.Start() + if err != nil { + return "", false, err + } + + return truncateCommandOutput(stdout, maxBytes) } func GetLog(user string, gist string, skip string) (string, error) { @@ -228,7 +238,7 @@ func Push(gistTmpId string) error { } func DeleteRepository(user string, gist string) error { - return os.RemoveAll(filepath.Join(config.GetHomeDir(), "repos", strings.ToLower(user), gist)) + return os.RemoveAll(RepositoryPath(user, gist)) } func UpdateServerInfo(user string, gist string) error { @@ -239,7 +249,7 @@ func UpdateServerInfo(user string, gist string) error { return cmd.Run() } -func RPCRefs(user string, gist string, service string) ([]byte, error) { +func RPC(user string, gist string, service string) ([]byte, error) { repositoryPath := RepositoryPath(user, gist) cmd := exec.Command("git", service, "--stateless-rpc", "--advertise-refs", ".") diff --git a/internal/git/output_parser.go b/internal/git/output_parser.go new file mode 100644 index 0000000..111ee31 --- /dev/null +++ b/internal/git/output_parser.go @@ -0,0 +1,39 @@ +package git + +import ( + "bytes" + "io" +) + +func truncateCommandOutput(out io.Reader, maxBytes int64) (string, bool, error) { + var ( + buf []byte + err error + ) + + if maxBytes < 0 { + // read entire output + buf, err = io.ReadAll(out) + if err != nil { + return "", false, err + } + return string(buf), false, nil + } + + // read up to maxBytes bytes + buf = make([]byte, maxBytes) + n, err := io.ReadFull(out, buf) + if err != nil && err != io.EOF && err != io.ErrUnexpectedEOF { + return "", false, err + } + bytesRead := int64(n) + + // find index of last newline character + lastNewline := bytes.LastIndexByte(buf, '\n') + if lastNewline >= 0 { + // truncate buffer to exclude last line + buf = buf[:lastNewline] + } + + return string(buf), bytesRead == maxBytes, nil +} diff --git a/internal/models/gist.go b/internal/models/gist.go index 4be1d3b..3c1c0d6 100644 --- a/internal/models/gist.go +++ b/internal/models/gist.go @@ -2,6 +2,8 @@ package models import ( "gorm.io/gorm" + "opengist/internal/git" + "os/exec" "time" ) @@ -24,14 +26,13 @@ type Gist struct { Likes []User `gorm:"many2many:likes;constraint:OnUpdate:CASCADE,OnDelete:CASCADE"` Forked *Gist `gorm:"foreignKey:ForkedID;constraint:OnUpdate:CASCADE,OnDelete:SET NULL"` ForkedID uint - - Files []File `gorm:"-"` } type File struct { Filename string `validate:"excludes=\x2f,excludes=\x5c,max=50"` OldFilename string `validate:"excludes=\x2f,excludes=\x5c,max=50"` Content string `validate:"required"` + Truncated bool } type Commit struct { @@ -186,6 +187,96 @@ func (gist *Gist) CanWrite(user *User) bool { return !(user == nil) && (gist.UserID == user.ID) } +func (gist *Gist) InitRepository() error { + return git.InitRepository(gist.User.Username, gist.Uuid) +} + +func (gist *Gist) DeleteRepository() error { + return git.DeleteRepository(gist.User.Username, gist.Uuid) +} + +func (gist *Gist) Files(revision string) ([]*File, error) { + var files []*File + filesStr, err := git.GetFilesOfRepository(gist.User.Username, gist.Uuid, revision) + if err != nil { + // if the revision or the file do not exist + + if exiterr, ok := err.(*exec.ExitError); ok && exiterr.ExitCode() == 128 { + return nil, nil + } + + return nil, err + } + + for _, fileStr := range filesStr { + file, err := gist.File(revision, fileStr, true) + if err != nil { + return nil, err + } + files = append(files, file) + } + return files, err +} + +func (gist *Gist) File(revision string, filename string, truncate bool) (*File, error) { + content, truncated, err := git.GetFileContent(gist.User.Username, gist.Uuid, revision, filename, truncate) + + // if the revision or the file do not exist + if exiterr, ok := err.(*exec.ExitError); ok && exiterr.ExitCode() == 128 { + return nil, nil + } + + return &File{ + Filename: filename, + Content: content, + Truncated: truncated, + }, err +} + +func (gist *Gist) Log(skip string) error { + _, err := git.GetLog(gist.User.Username, gist.Uuid, skip) + + return err +} + +func (gist *Gist) NbCommits() (string, error) { + return git.GetNumberOfCommitsOfRepository(gist.User.Username, gist.Uuid) +} + +func (gist *Gist) AddAndCommitFiles(files *[]File) error { + if err := git.CloneTmp(gist.User.Username, gist.Uuid, gist.Uuid); err != nil { + return err + } + + for _, file := range *files { + if err := git.SetFileContent(gist.Uuid, file.Filename, file.Content); err != nil { + return err + } + } + + if err := git.AddAll(gist.Uuid); err != nil { + return err + } + + if err := git.Commit(gist.Uuid); err != nil { + return err + } + + return git.Push(gist.Uuid) +} + +func (gist *Gist) ForkClone(username string, uuid string) error { + return git.ForkClone(gist.User.Username, gist.Uuid, username, uuid) +} + +func (gist *Gist) UpdateServerInfo() error { + return git.UpdateServerInfo(gist.User.Username, gist.Uuid) +} + +func (gist *Gist) RPC(service string) ([]byte, error) { + return git.RPC(gist.User.Username, gist.Uuid, service) +} + // -- DTO -- // type GistDTO struct { @@ -200,7 +291,6 @@ func (dto *GistDTO) ToGist() *Gist { Title: dto.Title, Description: dto.Description, Private: dto.Private, - Files: dto.Files, } } @@ -208,6 +298,5 @@ func (dto *GistDTO) ToExistingGist(gist *Gist) *Gist { gist.Title = dto.Title gist.Description = dto.Description gist.Private = dto.Private - gist.Files = dto.Files return gist } diff --git a/internal/web/admin.go b/internal/web/admin.go index a3474b2..e4db488 100644 --- a/internal/web/admin.go +++ b/internal/web/admin.go @@ -99,7 +99,7 @@ func adminGistDelete(ctx echo.Context) error { return errorRes(500, "Cannot retrieve gist", err) } - if err = git.DeleteRepository(gist.User.Username, gist.Uuid); err != nil { + if err = gist.DeleteRepository(); err != nil { return errorRes(500, "Cannot delete the repository", err) } diff --git a/internal/web/gist.go b/internal/web/gist.go index 76a97a0..2384756 100644 --- a/internal/web/gist.go +++ b/internal/web/gist.go @@ -4,7 +4,6 @@ import ( "archive/zip" "bytes" "errors" - "fmt" "github.com/google/uuid" "github.com/labstack/echo/v4" "gorm.io/gorm" @@ -53,7 +52,7 @@ func gistInit(next echo.HandlerFunc) echo.HandlerFunc { setData(ctx, "currentUrl", template.URL(ctx.Request().URL.Path)) - nbCommits, err := git.GetNumberOfCommitsOfRepository(userName, gistName) + nbCommits, err := gist.NbCommits() if err != nil { return errorRes(500, "Error fetching number of commits", err) } @@ -133,27 +132,19 @@ func allGists(ctx echo.Context) error { func gistIndex(ctx echo.Context) error { gist := getData(ctx, "gist").(*models.Gist) - userName := gist.User.Username - gistName := gist.Uuid revision := ctx.Param("revision") if revision == "" { revision = "HEAD" } - nbCommits := getData(ctx, "nbCommits") - files := make(map[string]string) - if nbCommits != "0" { - filesStr, err := git.GetFilesOfRepository(userName, gistName, revision) - if err != nil { - return errorRes(500, "Error fetching files from repository", err) - } - for _, file := range filesStr { - files[file], err = git.GetFileContent(userName, gistName, revision, file) - if err != nil { - return errorRes(500, "Error fetching file content from file "+file, err) - } - } + files, err := gist.Files(revision) + if err != nil { + return errorRes(500, "Error fetching files", err) + } + + if len(files) == 0 { + return notFound("Revision not found") } setData(ctx, "page", "code") @@ -161,7 +152,6 @@ func gistIndex(ctx echo.Context) error { setData(ctx, "files", files) setData(ctx, "revision", revision) setData(ctx, "htmlTitle", gist.Title) - return html(ctx, "gist.html") } @@ -256,7 +246,6 @@ func processCreate(ctx echo.Context) error { } if err := ctx.Bind(dto); err != nil { - fmt.Println(err) return errorRes(400, "Cannot bind data", err) } @@ -286,18 +275,10 @@ func processCreate(ctx echo.Context) error { if isCreate { return html(ctx, "create.html") } else { - files := make(map[string]string) - filesStr, err := git.GetFilesOfRepository(gist.User.Username, gist.Uuid, "HEAD") + files, err := gist.Files("HEAD") if err != nil { - return errorRes(500, "Error fetching files from repository", err) + return errorRes(500, "Error fetching files", err) } - for _, file := range filesStr { - files[file], err = git.GetFileContent(gist.User.Username, gist.Uuid, "HEAD", file) - if err != nil { - return errorRes(500, "Error fetching file content from file "+file, err) - } - } - setData(ctx, "files", files) return html(ctx, "edit.html") } @@ -310,7 +291,7 @@ func processCreate(ctx echo.Context) error { } user := getUserLogged(ctx) - gist.NbFiles = len(gist.Files) + gist.NbFiles = len(dto.Files) if isCreate { uuidGist, err := uuid.NewRandom() @@ -320,6 +301,7 @@ func processCreate(ctx echo.Context) error { gist.Uuid = strings.Replace(uuidGist.String(), "-", "", -1) gist.UserID = user.ID + gist.User = *user } if gist.Title == "" { @@ -330,41 +312,23 @@ func processCreate(ctx echo.Context) error { } } - if len(gist.Files) > 0 { - split := strings.Split(gist.Files[0].Content, "\n") + if len(dto.Files) > 0 { + split := strings.Split(dto.Files[0].Content, "\n") if len(split) > 10 { gist.Preview = strings.Join(split[:10], "\n") } else { - gist.Preview = gist.Files[0].Content + gist.Preview = dto.Files[0].Content } - gist.PreviewFilename = gist.Files[0].Filename + gist.PreviewFilename = dto.Files[0].Filename } - if err = git.InitRepository(user.Username, gist.Uuid); err != nil { + if err = gist.InitRepository(); err != nil { return errorRes(500, "Error creating the repository", err) } - if err = git.CloneTmp(user.Username, gist.Uuid, gist.Uuid); err != nil { - return errorRes(500, "Error cloning the repository", err) - } - - for _, file := range gist.Files { - if err = git.SetFileContent(gist.Uuid, file.Filename, file.Content); err != nil { - return errorRes(500, "Error setting file content for file "+file.Filename, err) - } - } - - if err = git.AddAll(gist.Uuid); err != nil { - return errorRes(500, "Error adding files to the repository", err) - } - - if err = git.Commit(gist.Uuid); err != nil { - return errorRes(500, "Error committing files to the local repository", err) - } - - if err = git.Push(gist.Uuid); err != nil { - return errorRes(500, "Error pushing the local repository", err) + if err = gist.AddAndCommitFiles(&dto.Files); err != nil { + return errorRes(500, "Error adding and commiting files", err) } if isCreate { @@ -395,7 +359,7 @@ func toggleVisibility(ctx echo.Context) error { func deleteGist(ctx echo.Context) error { var gist = getData(ctx, "gist").(*models.Gist) - err := git.DeleteRepository(gist.User.Username, gist.Uuid) + err := gist.DeleteRepository() if err != nil { return errorRes(500, "Error deleting the repository", err) } @@ -472,7 +436,7 @@ func fork(ctx echo.Context) error { return errorRes(500, "Error forking the gist in database", err) } - if err = git.ForkClone(gist.User.Username, gist.Uuid, currentUser.Username, newGist.Uuid); err != nil { + if err = gist.ForkClone(currentUser.Username, newGist.Uuid); err != nil { return errorRes(500, "Error cloning the repository while forking", err) } if err = gist.IncrementForkCount(); err != nil { @@ -486,38 +450,26 @@ func fork(ctx echo.Context) error { func rawFile(ctx echo.Context) error { gist := getData(ctx, "gist").(*models.Gist) - fileContent, err := git.GetFileContent( - gist.User.Username, - gist.Uuid, - ctx.Param("revision"), - ctx.Param("file")) + file, err := gist.File(ctx.Param("revision"), ctx.Param("file"), false) + if err != nil { return errorRes(500, "Error getting file content", err) } - filebytes := []byte(fileContent) - - if len(filebytes) == 0 { + if file == nil { return notFound("File not found") } - return plainText(ctx, 200, string(filebytes)) + return plainText(ctx, 200, file.Content) } func edit(ctx echo.Context) error { var gist = getData(ctx, "gist").(*models.Gist) - files := make(map[string]string) - filesStr, err := git.GetFilesOfRepository(gist.User.Username, gist.Uuid, "HEAD") + files, err := gist.Files("HEAD") if err != nil { return errorRes(500, "Error fetching files from repository", err) } - for _, file := range filesStr { - files[file], err = git.GetFileContent(gist.User.Username, gist.Uuid, "HEAD", file) - if err != nil { - return errorRes(500, "Error fetching file content from file "+file, err) - } - } setData(ctx, "files", files) setData(ctx, "htmlTitle", "Edit "+gist.Title) @@ -529,29 +481,24 @@ func downloadZip(ctx echo.Context) error { var gist = getData(ctx, "gist").(*models.Gist) var revision = ctx.Param("revision") - files := make(map[string]string) - filesStr, err := git.GetFilesOfRepository(gist.User.Username, gist.Uuid, revision) + files, err := gist.Files(revision) if err != nil { return errorRes(500, "Error fetching files from repository", err) } - - for _, file := range filesStr { - files[file], err = git.GetFileContent(gist.User.Username, gist.Uuid, revision, file) - if err != nil { - return errorRes(500, "Error fetching file content from file "+file, err) - } + if len(files) == 0 { + return notFound("No files found in this revision") } zipFile := new(bytes.Buffer) zipWriter := zip.NewWriter(zipFile) - for fileName, fileContent := range files { - f, err := zipWriter.Create(fileName) + for _, file := range files { + f, err := zipWriter.Create(file.Filename) if err != nil { return errorRes(500, "Error adding a file the to the zip archive", err) } - _, err = f.Write([]byte(fileContent)) + _, err = f.Write([]byte(file.Content)) if err != nil { return errorRes(500, "Error adding file content the to the zip archive", err) } diff --git a/internal/web/git-http.go b/internal/web/git-http.go index 9211443..bd49079 100644 --- a/internal/web/git-http.go +++ b/internal/web/git-http.go @@ -152,13 +152,13 @@ func infoRefs(ctx echo.Context) error { service = strings.TrimPrefix(serviceType, "git-") if service != "upload-pack" && service != "receive-pack" { - if err := git.UpdateServerInfo(gist.User.Username, gist.Uuid); err != nil { + if err := gist.UpdateServerInfo(); err != nil { return errorRes(500, "Cannot update server info", err) } return sendFile(ctx, "text/plain; charset=utf-8") } - refs, err := git.RPCRefs(gist.User.Username, gist.Uuid, service) + refs, err := gist.RPC(service) if err != nil { return errorRes(500, "Cannot run git "+service, err) } diff --git a/templates/pages/edit.html b/templates/pages/edit.html index 99a539a..f2cf3d1 100644 --- a/templates/pages/edit.html +++ b/templates/pages/edit.html @@ -53,11 +53,11 @@
- {{ range $filename, $content := .files }} + {{ range $file := .files }}

- +

- +
{{ end }}
diff --git a/templates/pages/gist.html b/templates/pages/gist.html index d3022a5..4a5c22f 100644 --- a/templates/pages/gist.html +++ b/templates/pages/gist.html @@ -2,30 +2,35 @@ {{ template "gist_header" .}} {{ if .files }}
- {{ range $filename, $content := .files }} + {{ range $file := .files }}
- {{ $filename }} + {{ $file.Filename }} - Raw - + Raw +
+ {{ if $file.Truncated }} +
+ This file has been truncated. View the full file. +
+ {{ end }}
- {{ if isMarkdown $filename }} -
{{ $content }}
+ {{ if isMarkdown $file.Filename }} +
{{ $file.Content }}
{{ else }} - {{ $fileslug := slug $filename }} - + {{ $fileslug := slug $file.Filename }} +
{{ $ii := "1" }} {{ $i := toInt $ii }} - {{ range $line := lines $content }}{{ $i = inc $i }}{{ end }} + {{ range $line := lines $file.Content }}{{ $i = inc $i }}{{ end }}
{{$i}}{{ $line }}
{{$i}}{{ $line }}
{{ end }}