From 17237713a179d299a3fd0c6279fa29e74f9d189d Mon Sep 17 00:00:00 2001 From: Thomas Miceli <27960254+thomiceli@users.noreply.github.com> Date: Fri, 20 Sep 2024 16:01:09 +0200 Subject: [PATCH] Add Postgres and MySQL databases support (#335) --- .github/workflows/go.yml | 56 +++++- Makefile | 3 +- README.md | 2 +- config.yml | 8 +- docs/.vitepress/config.mts | 8 +- docs/configuration/databases/mysql.md | 49 ++++++ docs/configuration/databases/postgresql.md | 48 +++++ docs/configuration/databases/sqlite.md | 39 +++++ docs/introduction.md | 2 +- go.mod | 12 +- go.sum | 19 +- internal/cli/hook.go | 4 +- internal/cli/main.go | 5 +- internal/config/config.go | 7 +- internal/db/admin_setting.go | 10 +- internal/db/db.go | 193 ++++++++++++++++++--- internal/db/invitation.go | 20 ++- internal/db/migration.go | 14 +- internal/db/user.go | 2 +- internal/ssh/git_ssh.go | 2 +- internal/web/admin.go | 3 + internal/web/test/server.go | 28 ++- templates/pages/admin_config.html | 4 +- 23 files changed, 479 insertions(+), 59 deletions(-) create mode 100644 docs/configuration/databases/mysql.md create mode 100644 docs/configuration/databases/postgresql.md create mode 100644 docs/configuration/databases/sqlite.md diff --git a/.github/workflows/go.yml b/.github/workflows/go.yml index cb38328..59c7dec 100644 --- a/.github/workflows/go.yml +++ b/.github/workflows/go.yml @@ -1,15 +1,15 @@ name: "Go CI" on: push: - branches: - - master - - 'dev-*' + workflow_dispatch: + pull_request: paths-ignore: - '**.yml' jobs: lint: + name: Lint runs-on: ubuntu-latest steps: - name: Checkout @@ -31,6 +31,7 @@ jobs: run: make fmt check_changes check: + name: Check runs-on: ubuntu-latest steps: - name: Checkout @@ -47,12 +48,58 @@ jobs: - name: Check translations run: make check-tr + test-db: + name: Test + strategy: + fail-fast: false + matrix: + os: ["ubuntu-latest"] + go: ["1.22"] + database: [postgres, mysql] + include: + - database: postgres + image: postgres:16 + port: 5432:5432 + - database: mysql + image: mysql:8 + port: 3306:3306 + + runs-on: ${{ matrix.os }} + services: + database: + image: ${{ matrix.image }} + ports: + - ${{ matrix.port }} + env: + POSTGRES_PASSWORD: opengist + POSTGRES_DB: opengist_test + MYSQL_ROOT_PASSWORD: opengist + MYSQL_DATABASE: opengist_test + options: >- + --health-cmd ${{ matrix.database == 'postgres' && 'pg_isready' || '"mysqladmin ping"' }} + --health-interval 10s + --health-timeout 5s + --health-retries 5 + steps: + - name: Checkout + uses: actions/checkout@v3 + + - name: Set up Go ${{ matrix.go }} + uses: actions/setup-go@v4 + with: + go-version: ${{ matrix.go }} + + - name: Run tests + run: make test TEST_DB_TYPE=${{ matrix.database }} + test: + name: Test strategy: fail-fast: false matrix: os: ["ubuntu-latest", "macOS-latest", "windows-latest"] go: ["1.22"] + database: ["sqlite"] runs-on: ${{ matrix.os }} steps: - name: Checkout @@ -64,5 +111,4 @@ jobs: go-version: ${{ matrix.go }} - name: Run tests - run: make test - + run: make test TEST_DB_TYPE=${{ matrix.database }} \ No newline at end of file diff --git a/Makefile b/Makefile index 1d880d4..f8a3425 100644 --- a/Makefile +++ b/Makefile @@ -4,6 +4,7 @@ BINARY_NAME := opengist GIT_TAG := $(shell git describe --tags) VERSION_PKG := github.com/thomiceli/opengist/internal/config.OpengistVersion +TEST_DB_TYPE ?= sqlite all: clean install build @@ -72,7 +73,7 @@ fmt: @go fmt ./... test: - @go test ./... -p 1 + @OPENGIST_TEST_DB=$(TEST_DB_TYPE) go test ./... -p 1 check-tr: @bash ./scripts/check-translations.sh \ No newline at end of file diff --git a/README.md b/README.md index 753af31..eb63eb3 100644 --- a/README.md +++ b/README.md @@ -28,7 +28,7 @@ It is similiar to [GitHub Gist](https://gist.github.com/), but open-source and c * OAuth2 login with GitHub, GitLab, Gitea, and OpenID Connect * Restrict or unrestrict snippets visibility to anonymous users * Docker support -* [More...](/docs/index.md#features) +* [More...](/docs/introduction.md#features) ## Quick start diff --git a/config.yml b/config.yml index 2942714..fa508c4 100644 --- a/config.yml +++ b/config.yml @@ -14,8 +14,11 @@ external-url: # Directory where Opengist will store its data. Default: ~/.opengist/ opengist-home: -# Name of the SQLite database file. Default: opengist.db -db-filename: opengist.db +# URI of the database. Default: opengist.db (SQLite) +# SQLite: file name +# PostgreSQL: postgres://user:password@host:port/database +# MySQL/MariaDB: mysql://user:password@host:port/database +db-uri: opengist.db # Enable or disable the code search index (either `true` or `false`). Default: true index.enabled: true @@ -29,6 +32,7 @@ git.default-branch: # Set the journal mode for SQLite. Default: WAL # See https://www.sqlite.org/pragma.html#pragma_journal_mode +# For SQLite databases only. sqlite.journal-mode: WAL diff --git a/docs/.vitepress/config.mts b/docs/.vitepress/config.mts index 353abc4..4849d21 100644 --- a/docs/.vitepress/config.mts +++ b/docs/.vitepress/config.mts @@ -36,11 +36,17 @@ export default defineConfig({ { text: 'Configuration', base: '/docs/configuration', items: [ {text: 'Configure Opengist', link: '/configure'}, - {text: 'Admin panel', link: '/admin-panel'}, + {text: 'Databases', items: [ + {text: 'SQLite', link: '/databases/sqlite'}, + {text: 'PostgreSQL', link: '/databases/postgresql'}, + {text: 'MySQL', link: '/databases/mysql'}, + ], collapsed: true + }, {text: 'OAuth Providers', link: '/oauth-providers'}, {text: 'Custom assets', link: '/custom-assets'}, {text: 'Custom links', link: '/custom-links'}, {text: 'Cheat Sheet', link: '/cheat-sheet'}, + {text: 'Admin panel', link: '/admin-panel'}, ], collapsed: false }, { diff --git a/docs/configuration/databases/mysql.md b/docs/configuration/databases/mysql.md new file mode 100644 index 0000000..589043d --- /dev/null +++ b/docs/configuration/databases/mysql.md @@ -0,0 +1,49 @@ +# Using MySQL/MariaDB + +To use MySQL/MariaDB as the database backend, you need to set the database URI configuration to the connection string of your MySQL/MariaDB database with this format : + +`mysql://:@:/` + +#### YAML +```yaml +# Example +db-uri: mysql://root:passwd@localhost:3306/opengist_db +``` + +#### Environment variable +```sh +# Example +OG_DB_URI=mysql://root:passwd@localhost:3306/opengist_db +``` + +### Docker Compose +```yml +version: "3" + +services: + opengist: + image: ghcr.io/thomiceli/opengist:1 + container_name: opengist + restart: unless-stopped + depends_on: + - mysql + ports: + - "6157:6157" + - "2222:2222" + volumes: + - "$HOME/.opengist:/opengist" + environment: + OG_DB_URI: mysql://opengist:secret@mysql:3306/opengist_db + # other configuration options + + mysql: + image: mysql:8.4 + restart: unless-stopped + volumes: + - "./opengist-database:/var/lib/mysql" + environment: + MYSQL_USER: opengist + MYSQL_PASSWORD: secret + MYSQL_DATABASE: opengist_db + MYSQL_ROOT_PASSWORD: rootsecret +``` \ No newline at end of file diff --git a/docs/configuration/databases/postgresql.md b/docs/configuration/databases/postgresql.md new file mode 100644 index 0000000..2c7cb98 --- /dev/null +++ b/docs/configuration/databases/postgresql.md @@ -0,0 +1,48 @@ +# Using PostgreSQL + +To use PostgreSQL as the database backend, you need to set the database URI configuration to the connection string of your PostgreSQL database with this format : + +`postgres://:@:/` + +#### YAML +```yaml +# Example +db-uri: postgres://postgres:passwd@localhost:5432/opengist_db +``` + +#### Environment variable +```sh +# Example +OG_DB_URI=postgres://postgres:passwd@localhost:5432/opengist_db +``` + +### Docker Compose +```yml +version: "3" + +services: + opengist: + image: ghcr.io/thomiceli/opengist:1 + container_name: opengist + restart: unless-stopped + depends_on: + - postgres + ports: + - "6157:6157" + - "2222:2222" + volumes: + - "$HOME/.opengist:/opengist" + environment: + OG_DB_URI: postgres://opengist:secret@postgres:5432/opengist_db + # other configuration options + + postgres: + image: postgres:16.4 + restart: unless-stopped + volumes: + - "./opengist-database:/var/lib/postgresql/data" + environment: + POSTGRES_USER: opengist + POSTGRES_PASSWORD: secret + POSTGRES_DB: opengist_db +``` \ No newline at end of file diff --git a/docs/configuration/databases/sqlite.md b/docs/configuration/databases/sqlite.md new file mode 100644 index 0000000..cadf90a --- /dev/null +++ b/docs/configuration/databases/sqlite.md @@ -0,0 +1,39 @@ +# Using SQLite + +By default, Opengist uses SQLite as the database backend. + +Because SQLite is a file-based database, there is not much configuration to tweak. + +The configuration `db-uri`/`OG_DB_URI` refers to the path of the SQLite database file relative in the `$opengist-home/` directory (default `opengist.db`), +although it can be left untouched. + +The SQLite journal mode is set to [`WAL` (Write-Ahead Logging)](https://www.sqlite.org/pragma.html#pragma_journal_mode) by default and can be changed. + +#### YAML +```yaml +sqlite.journal-mode: WAL +``` + +#### Environment variable +```sh +OG_SQLITE_JOURNAL_MODE=WAL +``` + +### Docker Compose +```yml +version: "3" + +services: + opengist: + image: ghcr.io/thomiceli/opengist:1 + container_name: opengist + restart: unless-stopped + ports: + - "6157:6157" # HTTP port + - "2222:2222" # SSH port, can be removed if you don't use SSH + volumes: + - "$HOME/.opengist:/opengist" + environment: + OG_SQLITE_JOURNAL_MODE: WAL + # other configuration options +``` \ No newline at end of file diff --git a/docs/introduction.md b/docs/introduction.md index 0bf7814..673c448 100644 --- a/docs/introduction.md +++ b/docs/introduction.md @@ -31,7 +31,7 @@ Written in [Go](https://go.dev), Opengist aims to be fast and easy to deploy. * delete users/gists; * clean database/filesystem by syncing gists * run `git gc` for all repositories -* SQLite database +* SQLite/PostgreSQL/MySQL database * Logging * Docker support diff --git a/go.mod b/go.mod index 1d50e13..ea3a525 100644 --- a/go.mod +++ b/go.mod @@ -7,7 +7,6 @@ require ( github.com/alecthomas/chroma/v2 v2.14.0 github.com/blevesearch/bleve/v2 v2.4.0 github.com/dustin/go-humanize v1.0.1 - github.com/glebarez/go-sqlite v1.22.0 github.com/glebarez/sqlite v1.11.0 github.com/go-playground/validator/v10 v10.21.0 github.com/google/uuid v1.6.0 @@ -26,7 +25,9 @@ require ( golang.org/x/crypto v0.23.0 golang.org/x/text v0.15.0 gopkg.in/yaml.v3 v3.0.1 - gorm.io/gorm v1.25.10 + gorm.io/driver/mysql v1.5.7 + gorm.io/driver/postgres v1.5.9 + gorm.io/gorm v1.25.12 ) require ( @@ -53,8 +54,10 @@ require ( github.com/davecgh/go-spew v1.1.1 // indirect github.com/dlclark/regexp2 v1.11.0 // indirect github.com/gabriel-vasile/mimetype v1.4.4 // indirect + github.com/glebarez/go-sqlite v1.22.0 // indirect github.com/go-playground/locales v0.14.1 // indirect github.com/go-playground/universal-translator v0.18.1 // indirect + github.com/go-sql-driver/mysql v1.7.0 // indirect github.com/golang-jwt/jwt v3.2.2+incompatible // indirect github.com/golang/geo v0.0.0-20230421003525-6adc56603217 // indirect github.com/golang/protobuf v1.5.4 // indirect @@ -62,6 +65,10 @@ require ( github.com/gorilla/mux v1.8.1 // indirect github.com/hashicorp/go-immutable-radix v1.3.1 // indirect github.com/hashicorp/golang-lru v1.0.2 // indirect + github.com/jackc/pgpassfile v1.0.0 // indirect + github.com/jackc/pgservicefile v0.0.0-20221227161230-091c0ba34f0a // indirect + github.com/jackc/pgx/v5 v5.5.5 // indirect + github.com/jackc/puddle/v2 v2.2.1 // indirect github.com/jinzhu/inflection v1.0.0 // indirect github.com/jinzhu/now v1.1.5 // indirect github.com/json-iterator/go v1.1.12 // indirect @@ -85,6 +92,7 @@ require ( go.etcd.io/bbolt v1.3.10 // indirect golang.org/x/net v0.25.0 // indirect golang.org/x/oauth2 v0.20.0 // indirect + golang.org/x/sync v0.5.0 // indirect golang.org/x/sys v0.20.0 // indirect golang.org/x/time v0.5.0 // indirect google.golang.org/protobuf v1.34.1 // indirect diff --git a/go.sum b/go.sum index 2142749..39fbb83 100644 --- a/go.sum +++ b/go.sum @@ -82,6 +82,8 @@ github.com/go-playground/universal-translator v0.18.1 h1:Bcnm0ZwsGyWbCzImXv+pAJn github.com/go-playground/universal-translator v0.18.1/go.mod h1:xekY+UJKNuX9WP91TpwSH2VMlDf28Uj24BCp08ZFTUY= github.com/go-playground/validator/v10 v10.21.0 h1:4fZA11ovvtkdgaeev9RGWPgc1uj3H8W+rNYyH/ySBb0= github.com/go-playground/validator/v10 v10.21.0/go.mod h1:dbuPbCMFw/DrkbEynArYaCwl3amGuJotoKCe95atGMM= +github.com/go-sql-driver/mysql v1.7.0 h1:ueSltNNllEqE3qcWBTD0iQd3IpL/6U+mJxLkazJ7YPc= +github.com/go-sql-driver/mysql v1.7.0/go.mod h1:OXbVy3sEdcQ2Doequ6Z5BW6fXNQTmx+9S1MCJN5yJMI= github.com/gobwas/httphead v0.1.0 h1:exrUm0f4YX0L7EBwZHuCF4GDp8aJfVeBrlLQrs6NqWU= github.com/gobwas/httphead v0.1.0/go.mod h1:O/RXo79gxV8G+RqlR/otEwx4Q36zl9rqC5u12GKvMCM= github.com/gobwas/pool v0.2.1 h1:xfeeEhW7pwmX8nuLVlqbzVc7udMDrwetjEv+TZIz1og= @@ -125,6 +127,14 @@ github.com/hashicorp/golang-lru v1.0.2 h1:dV3g9Z/unq5DpblPpw+Oqcv4dU/1omnb4Ok8iP github.com/hashicorp/golang-lru v1.0.2/go.mod h1:iADmTwqILo4mZ8BN3D2Q6+9jd8WM5uGBxy+E8yxSoD4= github.com/hexops/gotextdiff v1.0.3 h1:gitA9+qJrrTCsiCl7+kh75nPqQt1cx4ZkudSTLoUqJM= github.com/hexops/gotextdiff v1.0.3/go.mod h1:pSWU5MAI3yDq+fZBTazCSJysOMbxWL1BSow5/V2vxeg= +github.com/jackc/pgpassfile v1.0.0 h1:/6Hmqy13Ss2zCq62VdNG8tM1wchn8zjSGOBJ6icpsIM= +github.com/jackc/pgpassfile v1.0.0/go.mod h1:CEx0iS5ambNFdcRtxPj5JhEz+xB6uRky5eyVu/W2HEg= +github.com/jackc/pgservicefile v0.0.0-20221227161230-091c0ba34f0a h1:bbPeKD0xmW/Y25WS6cokEszi5g+S0QxI/d45PkRi7Nk= +github.com/jackc/pgservicefile v0.0.0-20221227161230-091c0ba34f0a/go.mod h1:5TJZWKEWniPve33vlWYSoGYefn3gLQRzjfDlhSJ9ZKM= +github.com/jackc/pgx/v5 v5.5.5 h1:amBjrZVmksIdNjxGW/IiIMzxMKZFelXbUoPNb+8sjQw= +github.com/jackc/pgx/v5 v5.5.5/go.mod h1:ez9gk+OAat140fv9ErkZDYFWmXLfV+++K0uAOiwgm1A= +github.com/jackc/puddle/v2 v2.2.1 h1:RhxXJtFG022u4ibrCSMSiu5aOq1i77R3OHKNJj77OAk= +github.com/jackc/puddle/v2 v2.2.1/go.mod h1:vriiEXHvEE654aYKXXjOvZM39qJ0q+azkZFrfEOc3H4= github.com/jinzhu/inflection v1.0.0 h1:K317FqzuhWc8YvSVlFMCCUb36O/S9MCKRDI7QkRKD/E= github.com/jinzhu/inflection v1.0.0/go.mod h1:h+uFLlag+Qp1Va5pdKtLDYj+kHp5pxUVkryuEj+Srlc= github.com/jinzhu/now v1.1.5 h1:/o9tlHleP7gOFmsnYNz3RGnqzefHA47wQpKrrdTIwXQ= @@ -236,8 +246,13 @@ gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EV gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= -gorm.io/gorm v1.25.10 h1:dQpO+33KalOA+aFYGlK+EfxcI5MbO7EP2yYygwh9h+s= -gorm.io/gorm v1.25.10/go.mod h1:hbnx/Oo0ChWMn1BIhpy1oYozzpM15i4YPuHDmfYtwg8= +gorm.io/driver/mysql v1.5.7 h1:MndhOPYOfEp2rHKgkZIhJ16eVUIRf2HmzgoPmh7FCWo= +gorm.io/driver/mysql v1.5.7/go.mod h1:sEtPWMiqiN1N1cMXoXmBbd8C6/l+TESwriotuRRpkDM= +gorm.io/driver/postgres v1.5.9 h1:DkegyItji119OlcaLjqN11kHoUgZ/j13E0jkJZgD6A8= +gorm.io/driver/postgres v1.5.9/go.mod h1:DX3GReXH+3FPWGrrgffdvCk3DQ1dwDPdmbenSkweRGI= +gorm.io/gorm v1.25.7/go.mod h1:hbnx/Oo0ChWMn1BIhpy1oYozzpM15i4YPuHDmfYtwg8= +gorm.io/gorm v1.25.12 h1:I0u8i2hWQItBq1WfE0o2+WuL9+8L21K9e2HHSTE/0f8= +gorm.io/gorm v1.25.12/go.mod h1:xh7N7RHfYlNc5EmcI/El95gXusucDrQnHXe0+CgWcLQ= modernc.org/cc/v4 v4.21.2 h1:dycHFB/jDc3IyacKipCNSDrjIC0Lm1hyoWOZTRR20Lk= modernc.org/cc/v4 v4.21.2/go.mod h1:HM7VJTZbUCR3rV8EYBi9wxnJ0ZBRiGE5OeGXNA0IsLQ= modernc.org/ccgo/v4 v4.17.8 h1:yyWBf2ipA0Y9GGz/MmCmi3EFpKgeS7ICrAFes+suEbs= diff --git a/internal/cli/hook.go b/internal/cli/hook.go index 94b3f9b..5af3b46 100644 --- a/internal/cli/hook.go +++ b/internal/cli/hook.go @@ -8,7 +8,6 @@ import ( "github.com/urfave/cli/v2" "io" "os" - "path/filepath" ) var CmdHook = cli.Command{ @@ -50,7 +49,8 @@ func initialize(ctx *cli.Context) { } config.InitLog() - if err := db.Setup(filepath.Join(config.GetHomeDir(), config.C.DBFilename), false); err != nil { + db.DeprecationDBFilename() + if err := db.Setup(config.C.DBUri, false); err != nil { log.Fatal().Err(err).Msg("Failed to initialize database in hooks") } } diff --git a/internal/cli/main.go b/internal/cli/main.go index c64939f..9b9ee3f 100644 --- a/internal/cli/main.go +++ b/internal/cli/main.go @@ -108,8 +108,9 @@ func Initialize(ctx *cli.Context) { if err := os.MkdirAll(filepath.Join(homePath, "custom"), 0755); err != nil { log.Fatal().Err(err).Send() } - log.Info().Msg("Database file: " + filepath.Join(homePath, config.C.DBFilename)) - if err := db.Setup(filepath.Join(homePath, config.C.DBFilename), false); err != nil { + + db.DeprecationDBFilename() + if err := db.Setup(config.C.DBUri, false); err != nil { log.Fatal().Err(err).Msg("Failed to initialize database") } diff --git a/internal/config/config.go b/internal/config/config.go index c47a429..1d65396 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -29,7 +29,10 @@ type config struct { LogOutput string `yaml:"log-output" env:"OG_LOG_OUTPUT"` ExternalUrl string `yaml:"external-url" env:"OG_EXTERNAL_URL"` OpengistHome string `yaml:"opengist-home" env:"OG_OPENGIST_HOME"` - DBFilename string `yaml:"db-filename" env:"OG_DB_FILENAME"` + + DBUri string `yaml:"db-uri" env:"OG_DB_URI"` + DBFilename string `yaml:"db-filename" env:"OG_DB_FILENAME"` // deprecated + IndexEnabled bool `yaml:"index.enabled" env:"OG_INDEX_ENABLED"` IndexDirname string `yaml:"index.dirname" env:"OG_INDEX_DIRNAME"` @@ -80,7 +83,7 @@ func configWithDefaults() (*config, error) { c.LogLevel = "warn" c.LogOutput = "stdout,file" c.OpengistHome = "" - c.DBFilename = "opengist.db" + c.DBUri = "opengist.db" c.IndexEnabled = true c.IndexDirname = "opengist.index" diff --git a/internal/db/admin_setting.go b/internal/db/admin_setting.go index c338203..88f4ccd 100644 --- a/internal/db/admin_setting.go +++ b/internal/db/admin_setting.go @@ -5,7 +5,7 @@ import ( ) type AdminSetting struct { - Key string `gorm:"uniqueIndex"` + Key string `gorm:"index:,unique"` Value string } @@ -49,7 +49,7 @@ func UpdateSetting(key string, value string) error { } func setSetting(key string, value string) error { - return db.Create(&AdminSetting{Key: key, Value: value}).Error + return db.FirstOrCreate(&AdminSetting{Key: key, Value: value}, &AdminSetting{Key: key}).Error } func initAdminSettings(settings map[string]string) error { @@ -64,9 +64,9 @@ func initAdminSettings(settings map[string]string) error { return nil } -type DBAuthInfo struct{} +type AuthInfo struct{} -func (auth DBAuthInfo) RequireLogin() (bool, error) { +func (auth AuthInfo) RequireLogin() (bool, error) { s, err := GetSetting(SettingRequireLogin) if err != nil { return true, err @@ -74,7 +74,7 @@ func (auth DBAuthInfo) RequireLogin() (bool, error) { return s == "1", nil } -func (auth DBAuthInfo) AllowGistsWithoutLogin() (bool, error) { +func (auth AuthInfo) AllowGistsWithoutLogin() (bool, error) { s, err := GetSetting(SettingAllowGistsWithoutLogin) if err != nil { return false, err diff --git a/internal/db/db.go b/internal/db/db.go index b6adb48..49f93b3 100644 --- a/internal/db/db.go +++ b/internal/db/db.go @@ -2,38 +2,133 @@ package db import ( "errors" + "fmt" + "github.com/glebarez/sqlite" + "gorm.io/driver/mysql" + "gorm.io/driver/postgres" + "gorm.io/gorm/logger" + "net/url" + "path/filepath" "slices" "strings" + "time" - msqlite "github.com/glebarez/go-sqlite" - "github.com/glebarez/sqlite" "github.com/rs/zerolog/log" "github.com/thomiceli/opengist/internal/config" "gorm.io/gorm" - "gorm.io/gorm/logger" ) var db *gorm.DB -func Setup(dbPath string, sharedCache bool) error { - var err error - journalMode := strings.ToUpper(config.C.SqliteJournalMode) +const ( + SQLite databaseType = iota + PostgreSQL + MySQL +) - if !slices.Contains([]string{"DELETE", "TRUNCATE", "PERSIST", "MEMORY", "WAL", "OFF"}, journalMode) { - log.Warn().Msg("Invalid SQLite journal mode: " + journalMode) +type databaseType int + +func (d databaseType) String() string { + return [...]string{"SQLite", "PostgreSQL", "MySQL"}[d] +} + +type databaseInfo struct { + Type databaseType + Host string + Port string + User string + Password string + Database string +} + +var DatabaseInfo *databaseInfo + +func parseDBURI(uri string) (*databaseInfo, error) { + info := &databaseInfo{} + + if !strings.Contains(uri, "://") { + info.Type = SQLite + if uri == "file::memory:" { + info.Database = "file::memory:" + return info, nil + } + info.Database = filepath.Join(config.GetHomeDir(), uri) + return info, nil + } + u, err := url.Parse(uri) + if err != nil { + return nil, fmt.Errorf("invalid URI: %v", err) } - sharedCacheStr := "" - if sharedCache { - sharedCacheStr = "&cache=shared" + switch u.Scheme { + case "postgres", "postgresql": + info.Type = PostgreSQL + case "mysql", "mariadb": + info.Type = MySQL + default: + return nil, fmt.Errorf("unknown database: %v", err) } - if db, err = gorm.Open(sqlite.Open(dbPath+"?_fk=true&_journal_mode="+journalMode+sharedCacheStr), &gorm.Config{ - Logger: logger.Default.LogMode(logger.Silent), - }); err != nil { + if u.Host != "" { + host, port, _ := strings.Cut(u.Host, ":") + info.Host = host + info.Port = port + } + + if u.User != nil { + info.User = u.User.Username() + info.Password, _ = u.User.Password() + } + + switch info.Type { + case PostgreSQL, MySQL: + info.Database = strings.TrimPrefix(u.Path, "/") + default: + return nil, fmt.Errorf("unknown database: %v", err) + } + + return info, nil +} + +func Setup(dbUri string, sharedCache bool) error { + dbInfo, err := parseDBURI(dbUri) + if err != nil { return err } + log.Info().Msgf("Setting up a %s database connection", dbInfo.Type) + var setupFunc func(databaseInfo, bool) error + switch dbInfo.Type { + case SQLite: + setupFunc = setupSQLite + case PostgreSQL: + setupFunc = setupPostgres + case MySQL: + setupFunc = setupMySQL + default: + return fmt.Errorf("unknown database type: %v", dbInfo.Type) + } + + maxAttempts := 60 + retryInterval := 1 * time.Second + + for attempt := 1; attempt <= maxAttempts; attempt++ { + err = setupFunc(*dbInfo, sharedCache) + if err == nil { + log.Info().Msg("Database connection established") + break + } + + if attempt < maxAttempts { + log.Warn().Err(err).Msgf("Failed to connect to database (attempt %d), retrying in %v...", attempt, retryInterval) + time.Sleep(retryInterval) + } else { + return err + } + } + + DatabaseInfo = dbInfo + if err = db.SetupJoinTable(&Gist{}, "Likes", &Like{}); err != nil { return err } @@ -46,7 +141,7 @@ func Setup(dbPath string, sharedCache bool) error { return err } - if err = ApplyMigrations(db); err != nil { + if err = applyMigrations(db, dbInfo); err != nil { return err } @@ -75,11 +170,7 @@ func CountAll(table interface{}) (int64, error) { } func IsUniqueConstraintViolation(err error) bool { - var sqliteErr *msqlite.Error - if errors.As(err, &sqliteErr) && sqliteErr.Code() == 2067 { - return true - } - return false + return errors.Is(err, gorm.ErrDuplicatedKey) } func Ping() error { @@ -90,3 +181,65 @@ func Ping() error { return sql.Ping() } + +func setupSQLite(dbInfo databaseInfo, sharedCache bool) error { + var err error + journalMode := strings.ToUpper(config.C.SqliteJournalMode) + + if !slices.Contains([]string{"DELETE", "TRUNCATE", "PERSIST", "MEMORY", "WAL", "OFF"}, journalMode) { + log.Warn().Msg("Invalid SQLite journal mode: " + journalMode) + } + + sharedCacheStr := "" + if sharedCache { + sharedCacheStr = "&cache=shared" + } + + db, err = gorm.Open(sqlite.Open(dbInfo.Database+"?_fk=true&_journal_mode="+journalMode+sharedCacheStr), &gorm.Config{ + Logger: logger.Default.LogMode(logger.Silent), + TranslateError: true, + }) + + return err +} + +func setupPostgres(dbInfo databaseInfo, sharedCache bool) error { + var err error + dsn := fmt.Sprintf("host=%s port=%s user=%s password=%s dbname=%s sslmode=disable", dbInfo.Host, dbInfo.Port, dbInfo.User, dbInfo.Password, dbInfo.Database) + + db, err = gorm.Open(postgres.Open(dsn), &gorm.Config{ + Logger: logger.Default.LogMode(logger.Silent), + TranslateError: true, + }) + + return err +} + +func setupMySQL(dbInfo databaseInfo, sharedCache bool) error { + var err error + dsn := fmt.Sprintf("%s:%s@tcp(%s:%s)/%s?charset=utf8mb4&parseTime=True&loc=Local", dbInfo.User, dbInfo.Password, dbInfo.Host, dbInfo.Port, dbInfo.Database) + + db, err = gorm.Open(mysql.New(mysql.Config{ + DSN: dsn, + DontSupportRenameIndex: true, + }), &gorm.Config{ + Logger: logger.Default.LogMode(logger.Silent), + TranslateError: true, + }) + + return err +} + +func DeprecationDBFilename() { + if config.C.DBFilename != "" { + log.Warn().Msg("The 'db-filename'/'OG_DB_FILENAME' configuration option is deprecated and will be removed in a future version. Please use 'db-uri'/'OG_DB_URI' instead.") + } + + if config.C.DBUri == "" { + config.C.DBUri = config.C.DBFilename + } +} + +func TruncateDatabase() error { + return db.Migrator().DropTable("likes", &User{}, "gists", &SSHKey{}, &AdminSetting{}, &Invitation{}) +} diff --git a/internal/db/invitation.go b/internal/db/invitation.go index f85fc83..f12f677 100644 --- a/internal/db/invitation.go +++ b/internal/db/invitation.go @@ -1,6 +1,7 @@ package db import ( + "fmt" "math/rand" "time" ) @@ -15,10 +16,21 @@ type Invitation struct { func GetAllInvitations() ([]*Invitation, error) { var invitations []*Invitation - err := db. - Order("(((expires_at >= strftime('%s', 'now')) AND ((nb_max <= 0) OR (nb_used < nb_max)))) desc"). - Order("id asc"). - Find(&invitations).Error + dialect := db.Dialector.Name() + query := db.Model(&Invitation{}) + + switch dialect { + case "sqlite": + query = query.Order("(((expires_at >= strftime('%s', 'now')) AND ((nb_max <= 0) OR (nb_used < nb_max)))) DESC") + case "postgres": + query = query.Order("(((expires_at >= EXTRACT(EPOCH FROM CURRENT_TIMESTAMP)) AND ((nb_max <= 0) OR (nb_used < nb_max)))) DESC") + case "mysql": + query = query.Order("(((expires_at >= UNIX_TIMESTAMP()) AND ((nb_max <= 0) OR (nb_used < nb_max)))) DESC") + default: + return nil, fmt.Errorf("unsupported database dialect: %s", dialect) + } + + err := query.Order("id ASC").Find(&invitations).Error return invitations, err } diff --git a/internal/db/migration.go b/internal/db/migration.go index 907e2b3..9510a87 100644 --- a/internal/db/migration.go +++ b/internal/db/migration.go @@ -11,7 +11,19 @@ type MigrationVersion struct { Version uint } -func ApplyMigrations(db *gorm.DB) error { +func applyMigrations(db *gorm.DB, dbInfo *databaseInfo) error { + switch dbInfo.Type { + case SQLite: + return applySqliteMigrations(db) + case PostgreSQL, MySQL: + return nil + default: + return fmt.Errorf("unknown database type: %s", dbInfo.Type) + } + +} + +func applySqliteMigrations(db *gorm.DB) error { // Create migration table if it doesn't exist if err := db.AutoMigrate(&MigrationVersion{}); err != nil { log.Fatal().Err(err).Msg("Error creating migration version table") diff --git a/internal/db/user.go b/internal/db/user.go index 37a0191..492bcbd 100644 --- a/internal/db/user.go +++ b/internal/db/user.go @@ -6,7 +6,7 @@ import ( type User struct { ID uint `gorm:"primaryKey"` - Username string `gorm:"uniqueIndex"` + Username string `gorm:"uniqueIndex,size:191"` Password string IsAdmin bool CreatedAt int64 diff --git a/internal/ssh/git_ssh.go b/internal/ssh/git_ssh.go index fe6f2b1..3c99994 100644 --- a/internal/ssh/git_ssh.go +++ b/internal/ssh/git_ssh.go @@ -39,7 +39,7 @@ func runGitCommand(ch ssh.Channel, gitCmd string, key string, ip string) error { return errors.New("gist not found") } - allowUnauthenticated, err := auth.ShouldAllowUnauthenticatedGistAccess(db.DBAuthInfo{}, true) + allowUnauthenticated, err := auth.ShouldAllowUnauthenticatedGistAccess(db.AuthInfo{}, true) if err != nil { return errors.New("internal server error") } diff --git a/internal/web/admin.go b/internal/web/admin.go index 22ef2d3..c49d778 100644 --- a/internal/web/admin.go +++ b/internal/web/admin.go @@ -161,6 +161,9 @@ func adminConfig(ctx echo.Context) error { setData(ctx, "htmlTitle", trH(ctx, "admin.configuration")+" - "+trH(ctx, "admin.admin_panel")) setData(ctx, "adminHeaderPage", "config") + setData(ctx, "dbtype", db.DatabaseInfo.Type.String()) + setData(ctx, "dbname", db.DatabaseInfo.Database) + return html(ctx, "admin_config.html") } diff --git a/internal/web/test/server.go b/internal/web/test/server.go index b10e40e..e6b5bd9 100644 --- a/internal/web/test/server.go +++ b/internal/web/test/server.go @@ -24,6 +24,8 @@ import ( "github.com/thomiceli/opengist/internal/web" ) +var databaseType string + type testServer struct { server *web.Server sessionCookie string @@ -132,6 +134,17 @@ func structToURLValues(s interface{}) url.Values { } func setup(t *testing.T) { + var databaseDsn string + databaseType = os.Getenv("OPENGIST_TEST_DB") + switch databaseType { + case "sqlite": + databaseDsn = "file::memory:" + case "postgres": + databaseDsn = "postgres://postgres:opengist@localhost:5432/opengist_test" + case "mysql": + databaseDsn = "mysql://root:opengist@localhost:3306/opengist_test" + } + _ = os.Setenv("OPENGIST_SKIP_GIT_HOOKS", "1") err := config.InitConfig("", io.Discard) @@ -155,9 +168,13 @@ func setup(t *testing.T) { err = os.MkdirAll(filepath.Join(homePath, "tmp", "repos"), 0755) require.NoError(t, err, "Could not create tmp repos directory") - err = db.Setup("file::memory:", true) + err = db.Setup(databaseDsn, true) require.NoError(t, err, "Could not initialize database") + if err != nil { + log.Fatal().Err(err).Msg("Could not initialize database") + } + err = memdb.Setup() require.NoError(t, err, "Could not initialize in memory database") @@ -168,10 +185,10 @@ func setup(t *testing.T) { func teardown(t *testing.T, s *testServer) { s.stop() - err := db.Close() - require.NoError(t, err, "Could not close database") + //err := db.Close() + //require.NoError(t, err, "Could not close database") - err = os.RemoveAll(path.Join(config.GetHomeDir(), "tests")) + err := os.RemoveAll(path.Join(config.GetHomeDir(), "tests")) require.NoError(t, err, "Could not remove repos directory") err = os.RemoveAll(path.Join(config.GetHomeDir(), "tmp", "repos")) @@ -180,6 +197,9 @@ func teardown(t *testing.T, s *testServer) { err = os.RemoveAll(path.Join(config.GetHomeDir(), "tmp", "sessions")) require.NoError(t, err, "Could not remove repos directory") + err = db.TruncateDatabase() + require.NoError(t, err, "Could not truncate database") + // err = os.RemoveAll(path.Join(config.C.OpengistHome, "testsindex")) // require.NoError(t, err, "Could not remove repos directory") diff --git a/templates/pages/admin_config.html b/templates/pages/admin_config.html index 1e628be..2cae5a6 100644 --- a/templates/pages/admin_config.html +++ b/templates/pages/admin_config.html @@ -17,11 +17,11 @@
Log output
{{ .c.LogOutput }}
External URL
{{ .c.ExternalUrl }}
Opengist home
{{ .c.OpengistHome }}
-
DB filename
{{ .c.DBFilename }}
+
Database type
{{ .dbtype }}{{ if eq .dbtype "SQLite" }} ({{ .c.SqliteJournalMode }}){{ end }}
+
Database name
{{ .dbname }}
Index Enabled
{{ .c.IndexEnabled }}
Index Dirname
{{ .c.IndexDirname }}
Git default branch
{{ .c.GitDefaultBranch }}
-
SQLite Journal Mode
{{ .c.SqliteJournalMode }}