mirror/opengist
1
0
Fork 0
mirror of https://github.com/thomiceli/opengist.git synced 2024-12-22 20:42:40 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

Create gists from git http server endpoint (#95)

Browse source
This commit is contained in:
Thomas Miceli 2023-09-09 19:39:57 +02:00 committed by GitHub
parent 977fc9db28
commit 46dea89b41
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
10 changed files with 276 additions and 25 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

3
go.mod
View file

@ -25,6 +25,9 @@ require (
github.com/gorilla/context v1.1.1 // indirect
github.com/gorilla/mux v1.6.2 // indirect
github.com/gorilla/securecookie v1.1.1 // indirect
github.com/hashicorp/go-immutable-radix v1.3.0 // indirect
github.com/hashicorp/go-memdb v1.3.4 // indirect
github.com/hashicorp/golang-lru v0.5.4 // indirect
github.com/jinzhu/inflection v1.0.0 // indirect
github.com/jinzhu/now v1.1.5 // indirect
github.com/labstack/gommon v0.4.0 // indirect

7
go.sum
View file

@ -128,8 +128,15 @@ github.com/gorilla/securecookie v1.1.1/go.mod h1:ra0sb63/xPlUeL+yeDciTfxMRAA+MP+
github.com/gorilla/sessions v1.1.1/go.mod h1:8KCfur6+4Mqcc6S0FEfKuN15Vl5MgXW92AE8ovaJD0w=
github.com/gorilla/sessions v1.2.1 h1:DHd3rPN5lE3Ts3D8rKkQ8x/0kqfeNmBAaiSi+o7FsgI=
github.com/gorilla/sessions v1.2.1/go.mod h1:dk2InVEVJ0sfLlnXv9EAgkf6ecYs/i80K/zI+bUmuGM=
github.com/hashicorp/go-immutable-radix v1.3.0 h1:8exGP7ego3OmkfksihtSouGMZ+hQrhxx+FVELeXpVPE=
github.com/hashicorp/go-immutable-radix v1.3.0/go.mod h1:0y9vanUI8NX6FsYoO3zeMjhV/C5i9g4Q3DwcSNZ4P60=
github.com/hashicorp/go-memdb v1.3.4 h1:XSL3NR682X/cVk2IeV0d70N4DZ9ljI885xAEU8IoK3c=
github.com/hashicorp/go-memdb v1.3.4/go.mod h1:uBTr1oQbtuMgd1SSGoR8YV27eT3sBHbYiNm53bMpgSg=
github.com/hashicorp/go-uuid v1.0.0/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro=
github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8=
github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8=
github.com/hashicorp/golang-lru v0.5.4 h1:YDjusn29QI/Das2iO9M0BHnIbxPeyuCHsjMW+lJfyTc=
github.com/hashicorp/golang-lru v0.5.4/go.mod h1:iADmTwqILo4mZ8BN3D2Q6+9jd8WM5uGBxy+E8yxSoD4=
github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc=
github.com/jarcoal/httpmock v0.0.0-20180424175123-9c70cfe4a1da/go.mod h1:ks+b9deReOc7jgqp+e7LuFiCBH6Rm5hL32cLcEAArb4=
github.com/jinzhu/inflection v1.0.0 h1:K317FqzuhWc8YvSVlFMCCUb36O/S9MCKRDI7QkRKD/E=

10
internal/db/gist.go
View file

@ -1,6 +1,7 @@
package db
import (
"github.com/labstack/echo/v4"
"github.com/thomiceli/opengist/internal/git"
"gorm.io/gorm"
"os/exec"
@ -190,6 +191,11 @@ func (gist *Gist) Update() error {
}
func (gist *Gist) Delete() error {
err := gist.DeleteRepository()
if err != nil {
return err
}
return db.Delete(&gist).Error
}
@ -260,6 +266,10 @@ func (gist *Gist) InitRepository() error {
return git.InitRepository(gist.User.Username, gist.Uuid)
}
func (gist *Gist) InitRepositoryViaNewPush(ctx echo.Context) error {
return git.InitRepositoryViaNewPush(gist.User.Username, gist.Uuid, ctx)
}
func (gist *Gist) DeleteRepository() error {
return git.DeleteRepository(gist.User.Username, gist.Uuid)
}

91
internal/git/commands.go
View file

@ -1,7 +1,9 @@
package git
import (
"bytes"
"fmt"
"github.com/labstack/echo/v4"
"github.com/rs/zerolog/log"
"github.com/thomiceli/opengist/internal/config"
"os"
@ -16,6 +18,23 @@ func RepositoryPath(user string, gist string) string {
return filepath.Join(config.GetHomeDir(), "repos", strings.ToLower(user), gist)
}
func RepositoryUrl(ctx echo.Context, user string, gist string) string {
httpProtocol := "http"
if ctx.Request().TLS != nil || ctx.Request().Header.Get("X-Forwarded-Proto") == "https" {
httpProtocol = "https"
}
var baseHttpUrl string
// if a custom external url is set, use it
if config.C.ExternalUrl != "" {
baseHttpUrl = config.C.ExternalUrl
} else {
baseHttpUrl = httpProtocol + "://" + ctx.Request().Host
}
return fmt.Sprintf("%s/%s/%s", baseHttpUrl, user, gist)
}
func TmpRepositoryPath(gistId string) string {
dirname := TmpRepositoriesPath()
return filepath.Join(dirname, gistId)
@ -40,7 +59,17 @@ func InitRepository(user string, gist string) error {
return err
}
return copyFiles(repositoryPath)
return createDotGitFiles(repositoryPath)
}
func InitRepositoryViaNewPush(user string, gist string, ctx echo.Context) error {
repositoryPath := RepositoryPath(user, gist)
if err := InitRepository(user, gist); err != nil {
return err
}
repositoryUrl := RepositoryUrl(ctx, user, gist)
return createDotGitHookFile(repositoryPath, "post-receive", fmt.Sprintf(postReceive, repositoryUrl, repositoryUrl))
}
func GetNumberOfCommitsOfRepository(user string, gist string) (string, error) {
@ -186,7 +215,7 @@ func ForkClone(userSrc string, gistSrc string, userDst string, gistDst string) e
return err
}
return copyFiles(repositoryPathDst)
return createDotGitFiles(repositoryPathDst)
}
func SetFileContent(gistTmpId string, filename string, content string) error {
@ -305,6 +334,26 @@ func GcRepos() error {
return err
}
func HasNoCommits(user string, gist string) (bool, error) {
repositoryPath := RepositoryPath(user, gist)
cmd := exec.Command("git", "rev-parse", "--all")
cmd.Dir = repositoryPath
var out bytes.Buffer
cmd.Stdout = &out
if err := cmd.Run(); err != nil {
return false, err
}
if out.String() == "" {
return true, nil // No commits exist
}
return false, nil // Commits exist
}
func GetGitVersion() (string, error) {
cmd := exec.Command("git", "--version")
stdout, err := cmd.Output()
@ -320,19 +369,27 @@ func GetGitVersion() (string, error) {
return versionFields[2], nil
}
func copyFiles(repositoryPath string) error {
func createDotGitFiles(repositoryPath string) error {
f1, err := os.OpenFile(filepath.Join(repositoryPath, "git-daemon-export-ok"), os.O_RDONLY|os.O_CREATE, 0644)
if err != nil {
return err
}
defer f1.Close()
preReceiveDst, err := os.OpenFile(filepath.Join(repositoryPath, "hooks", "pre-receive"), os.O_APPEND|os.O_WRONLY|os.O_CREATE, 0744)
if err = createDotGitHookFile(repositoryPath, "pre-receive", preReceive); err != nil {
return err
}
return nil
}
func createDotGitHookFile(repositoryPath string, hook string, content string) error {
preReceiveDst, err := os.OpenFile(filepath.Join(repositoryPath, "hooks", hook), os.O_APPEND|os.O_WRONLY|os.O_CREATE, 0744)
if err != nil {
return err
}
if _, err = preReceiveDst.WriteString(preReceive); err != nil {
if _, err = preReceiveDst.WriteString(content); err != nil {
return err
}
defer preReceiveDst.Close()
@ -346,6 +403,14 @@ disallowed_files=""
while read -r old_rev new_rev ref
do
if [ "$old_rev" = "0000000000000000000000000000000000000000" ]; then
# This is the first commit, so we check all the files in that commit
changed_files=$(git ls-tree -r --name-only "$new_rev")
else
# This is not the first commit, so we compare it with its predecessor
changed_files=$(git diff --name-only "$old_rev" "$new_rev")
fi
while IFS= read -r file
do
case $file in
@ -354,15 +419,29 @@ do
;;
esac
done <<EOF
$(git diff --name-only "$old_rev" "$new_rev")
$changed_files
EOF
done
if [ -n "$disallowed_files" ]; then
echo ""
echo "Pushing files in folders is not allowed:"
for file in $disallowed_files; do
echo " $file"
done
echo ""
exit 1
fi
`
const postReceive = `#!/bin/sh
echo ""
echo "Your new repository has been created here: %s"
echo ""
echo "If you want to keep working with your gist, you could set the remote URL via:"
echo "git remote set-url origin %s"
echo ""
rm -f $0
`

72
internal/memdb/memdb.go Normal file
View file

@ -0,0 +1,72 @@
package memdb
import "github.com/hashicorp/go-memdb"
import ogdb "github.com/thomiceli/opengist/internal/db"
var db *memdb.MemDB
type GistPush struct {
UserID uint
Gist *ogdb.Gist
}
func Setup() error {
var err error
schema := &memdb.DBSchema{
Tables: map[string]*memdb.TableSchema{
"gist_push": {
Name: "gist_push",
Indexes: map[string]*memdb.IndexSchema{
"id": {
Name: "id",
Unique: true,
Indexer: &memdb.UintFieldIndex{Field: "UserID"},
},
},
},
},
}
db, err = memdb.NewMemDB(schema)
if err != nil {
return err
}
return nil
}
func InsertGistPush(userId uint, gist *ogdb.Gist) error {
txn := db.Txn(true)
if err := txn.Insert("gist_push", &GistPush{
UserID: userId,
Gist: gist,
}); err != nil {
txn.Abort()
return err
}
txn.Commit()
return nil
}
func GetGistPushAndDelete(userId uint) (*GistPush, error) {
txn := db.Txn(true)
defer txn.Abort()
raw, err := txn.First("gist_push", "id", userId)
if err != nil {
return nil, err
}
if raw == nil {
return nil, nil
}
gistPush := raw.(*GistPush)
if err := txn.Delete("gist_push", gistPush); err != nil {
return nil, err
}
txn.Commit()
return gistPush, nil
}

13
internal/web/gist.go
View file

@ -104,6 +104,14 @@ func gistSoftInit(next echo.HandlerFunc) echo.HandlerFunc {
}
}
// gistNewPushInit has the same behavior as gistSoftInit but create a new gist empty instead
func gistNewPushSoftInit(next echo.HandlerFunc) echo.HandlerFunc {
return func(c echo.Context) error {
setData(c, "gist", new(db.Gist))
return next(c)
}
}
func allGists(ctx echo.Context) error {
var err error
var urlPage string
@ -428,11 +436,6 @@ func toggleVisibility(ctx echo.Context) error {
func deleteGist(ctx echo.Context) error {
var gist = getData(ctx, "gist").(*db.Gist)
err := gist.DeleteRepository()
if err != nil {
return errorRes(500, "Error deleting the repository", err)
}
if err := gist.Delete(); err != nil {
return errorRes(500, "Error deleting this gist", err)
}

94
internal/web/git_http.go
View file

@ -4,11 +4,15 @@ import (
"bytes"
"compress/gzip"
"encoding/base64"
"errors"
"fmt"
"github.com/google/uuid"
"github.com/labstack/echo/v4"
"github.com/rs/zerolog/log"
"github.com/thomiceli/opengist/internal/db"
"github.com/thomiceli/opengist/internal/git"
"github.com/thomiceli/opengist/internal/memdb"
"gorm.io/gorm"
"net/http"
"os"
"os/exec"
@ -47,8 +51,9 @@ func gitHttp(ctx echo.Context) error {
gist := getData(ctx, "gist").(*db.Gist)
isPushNew := strings.HasPrefix(ctx.Request().URL.Path, "/push/info/refs")
isPushReceive := strings.HasPrefix(ctx.Request().URL.Path, "/push/git-receive-pack")
isInfoRefs := strings.HasSuffix(route.gitUrl, "/info/refs$")
isPull := ctx.QueryParam("service") == "git-upload-pack" ||
strings.HasSuffix(ctx.Request().URL.Path, "git-upload-pack") ||
ctx.Request().Method == "GET" && !isInfoRefs
@ -61,7 +66,7 @@ func gitHttp(ctx echo.Context) error {
}
}
ctx.Set("repositoryPath", repositoryPath)
setData(ctx, "repositoryPath", repositoryPath)
// Shows basic auth if :
// - user wants to push the gist
@ -87,16 +92,70 @@ func gitHttp(ctx echo.Context) error {
return basicAuth(ctx)
}
if gist.ID == 0 {
return plainText(ctx, 404, "Check your credentials or make sure you have access to the Gist")
}
if ok, err := argon2id.verify(authPassword, gist.User.Password); !ok || gist.User.Username != authUsername {
if err != nil {
return errorRes(500, "Cannot verify password", err)
if !isPushNew && !isPushReceive {
if gist.ID == 0 {
return plainText(ctx, 404, "Check your credentials or make sure you have access to the Gist")
}
if ok, err := argon2id.verify(authPassword, gist.User.Password); !ok || gist.User.Username != authUsername {
if err != nil {
return errorRes(500, "Cannot verify password", err)
}
log.Warn().Msg("Invalid HTTP authentication attempt from " + ctx.RealIP())
return plainText(ctx, 404, "Check your credentials or make sure you have access to the Gist")
}
} else {
var user *db.User
if user, err = db.GetUserByUsername(authUsername); err != nil {
if !errors.Is(err, gorm.ErrRecordNotFound) {
return errorRes(500, "Cannot get user", err)
}
log.Warn().Msg("Invalid HTTP authentication attempt from " + ctx.RealIP())
return errorRes(401, "Invalid credentials", nil)
}
if ok, err := argon2id.verify(authPassword, user.Password); !ok {
if err != nil {
return errorRes(500, "Cannot check for password", err)
}
log.Warn().Msg("Invalid HTTP authentication attempt from " + ctx.RealIP())
return errorRes(401, "Invalid credentials", nil)
}
if isPushNew {
gist = new(db.Gist)
gist.UserID = user.ID
gist.User = *user
uuidGist, err := uuid.NewRandom()
if err != nil {
return errorRes(500, "Error creating an UUID", err)
}
gist.Uuid = strings.Replace(uuidGist.String(), "-", "", -1)
gist.Title = "gist:" + gist.Uuid
if err = gist.InitRepositoryViaNewPush(ctx); err != nil {
return errorRes(500, "Cannot init repository in the file system", err)
}
if err = gist.Create(); err != nil {
return errorRes(500, "Cannot init repository in database", err)
}
if err := memdb.InsertGistPush(user.ID, gist); err != nil {
return errorRes(500, "Cannot save the URL for the new Gist", err)
}
setData(ctx, "gist", gist)
} else {
gistFromMemdb, err := memdb.GetGistPushAndDelete(user.ID)
if err != nil {
return errorRes(500, "Cannot get the gist link from the in memory database", err)
}
gist := gistFromMemdb.Gist
setData(ctx, "gist", gist)
setData(ctx, "repositoryPath", git.RepositoryPath(gist.User.Username, gist.Uuid))
}
log.Warn().Msg("Invalid HTTP authentication attempt from " + ctx.RealIP())
return plainText(ctx, 404, "Check your credentials or make sure you have access to the Gist")
}
return route.handler(ctx)
@ -132,7 +191,7 @@ func pack(ctx echo.Context, serviceType string) error {
}
}
repositoryPath := ctx.Get("repositoryPath").(string)
repositoryPath := getData(ctx, "repositoryPath").(string)
var stderr bytes.Buffer
cmd := exec.Command("git", serviceType, "--stateless-rpc", repositoryPath)
@ -147,6 +206,15 @@ func pack(ctx echo.Context, serviceType string) error {
// updatedAt is updated only if serviceType is receive-pack
if serviceType == "receive-pack" {
gist := getData(ctx, "gist").(*db.Gist)
if hasNoCommits, err := git.HasNoCommits(gist.User.Username, gist.Uuid); err != nil {
return err
} else if hasNoCommits {
if err = gist.Delete(); err != nil {
return err
}
}
_ = gist.SetLastActiveNow()
_ = gist.UpdatePreviewAndCount()
}
@ -241,7 +309,7 @@ func basicAuthDecode(encoded string) (string, string, error) {
func sendFile(ctx echo.Context, contentType string) error {
gitFile := "/" + strings.Join(strings.Split(ctx.Request().URL.Path, "/")[3:], "/")
gitFile = path.Join(ctx.Get("repositoryPath").(string), gitFile)
gitFile = path.Join(getData(ctx, "repositoryPath").(string), gitFile)
fi, err := os.Stat(gitFile)
if os.IsNotExist(err) {
return errorRes(404, "File not found", nil)

4
internal/web/run.go
View file

@ -212,6 +212,10 @@ func Start() {
g2.PUT("/set-config", adminSetConfig)
}
if config.C.HttpGit {
e.Any("/push/*", gitHttp, gistNewPushSoftInit)
}
g1.GET("/all", allGists, checkRequireLogin)
g1.GET("/search", allGists, checkRequireLogin)
g1.GET("/:user", allGists, checkRequireLogin)

2
internal/web/util.go
View file

@ -167,7 +167,7 @@ func validateReservedKeywords(fl validator.FieldLevel) bool {
name := fl.Field().String()
restrictedNames := map[string]struct{}{}
for _, restrictedName := range []string{"assets", "register", "login", "logout", "settings", "admin-panel", "all", "search"} {
for _, restrictedName := range []string{"assets", "register", "login", "logout", "settings", "admin-panel", "all", "search", "push"} {
restrictedNames[restrictedName] = struct{}{}
}

5
opengist.go
View file

@ -7,6 +7,7 @@ import (
"github.com/thomiceli/opengist/internal/config"
"github.com/thomiceli/opengist/internal/db"
"github.com/thomiceli/opengist/internal/git"
"github.com/thomiceli/opengist/internal/memdb"
"github.com/thomiceli/opengist/internal/ssh"
"github.com/thomiceli/opengist/internal/web"
"os"
@ -54,6 +55,10 @@ func initialize() {
if err := db.Setup(filepath.Join(homePath, config.C.DBFilename)); err != nil {
log.Fatal().Err(err).Msg("Failed to initialize database")
}
if err := memdb.Setup(); err != nil {
log.Fatal().Err(err).Msg("Failed to initialize in memory database")
}
}
func main() {