Postgres DB + Metadata Server

.env.template

@@ -2,22 +2,31 @@ IS_DEV=True
 OAUTH_CLIENT_ID= # discord oauth client ID
 OAUTH_CLIENT_SECRET= # discord oauth client secret
 OAUTH_REDIRECT_URL=http://localhost:8730/auth/callback
+DEVICE_FLOW_VERIFICATION_URL=http://localhost:8730/auth/device
 PORT=8730
+MIN_LAUNCHER_VERSION=12.0.0 # minimum launcher version required to sync metadata
 AUTH_BOT_TOKEN= # bot used for reading the roles, contact dri0m for this i guess
 FLASHPOINT_SERVER_ID= 432708847304704010
 SECURECOOKIE_HASH_KEY_PREVIOUS=af00g0hjz0ue4w3hn4xe430gm56pgopx # used to encrypt cookies
 SECURECOOKIE_BLOCK_KEY_PREVIOUS=6v79vg3dkvjevd9wp6yjbxeig777w1ij # used to encrypt cookies
 SECURECOOKIE_HASH_KEY_CURRENT=wi117ggb3gligfv8xc3om79rsccqhing # used to encrypt cookies
 SECURECOOKIE_BLOCK_KEY_CURRENT=usqzaklwcegdwlwg0swt9xc3kh36shlb # used to encrypt cookies
 SESSION_EXPIRATION_SECONDS=2592000
+VALIDATOR_CONTAINER_NAME=fpfss-validator
 VALIDATOR_SERVER_URL=http://127.0.0.1:8371 # run the validator as well
+VALIDATOR_PORT=8371
 DDB_ROOT_USER=root
 DB_ROOT_PASSWORD=asdfghjkl
 DB_USER=fpfss
 DB_PASSWORD=asdfghjkl
 DB_IP=127.0.0.1
 DB_PORT=3306
 DB_NAME=fpfss
+POSTGRES_CONTAINER_NAME=fpfss-postgres
+POSTGRES_USER=fpfss
+POSTGRES_PASSWORD=password
+POSTGRES_HOST=localhost
+POSTGRES_PORT=5432
 NOTIFICATION_BOT_TOKEN= # shouldn't be needed for local dev
 NOTIFICATION_CHANNEL_ID=855479426079129630
 CURATION_FEED_CHANNEL_ID=856617787585462312
@@ -31,4 +40,12 @@ DB_CONTAINER_NAME=fpfssdb-local
 FLASHFREEZE_INGEST_DIR_FULL_PATH=/......../flashpoint-submission-system/files/flashfreeze-files/ingest
 FIXES_DIR_FULL_PATH=/......../flashpoint-submission-system/files/fixes-files
 SUBMISSIONS_DIR_FULL_PATH=/......../flashpoint-submission-system/files/submissions
-SUBMISSION_IMAGES_DIR_FULL_PATH=/......../flashpoint-submission-system/files/submissions-images
+SUBMISSION_IMAGES_DIR_FULL_PATH=/......../flashpoint-submission-system/files/submissions-images
+REPACK_DIR=/......../flashpoint-submission-system/files/temp
+SYSTEM_UID=123456789012345 # discord ID of the user used for actions not tied to a particular user (e.g for developer json imports)
+IMAGES_CDN=https://infinity.unstable.life/images
+IMAGES_CDN_COMPRESSED=False
+IMAGES_PATH=./files/live-images
+DATA_PACKS_PATH=./files/live-games
+DELETED_IMAGES_PATH=./files/deleted-images
+DELETED_DATA_PACKS_PATH=./files/deleted-games

.gitmodules

@@ -0,0 +1,3 @@
+[submodule "validation_bot"]
+	path = validator
+	url = https://github.com/FlashpointProject/Curation-Validation-Bot

Makefile

@@ -5,15 +5,37 @@ export $(shell sed 's/=.*//' .env)
 .PHONY: db migrate run
 
 db:
-	docker-compose -p ${DB_CONTAINER_NAME}  -f dc-db.yml down -v
+	docker-compose -p ${DB_CONTAINER_NAME}  -f dc-db.yml down
+	$(shell mkdir -p ${REPACK_DIR})
 	docker-compose -p ${DB_CONTAINER_NAME}  -f dc-db.yml up -d
 
+local:
+	docker-compose -p ${DB_CONTAINER_NAME}  -f dc-db.yml down
+	$(shell mkdir -p ${REPACK_DIR})
+	docker-compose -p ${DB_CONTAINER_NAME}  -f dc-db.yml up -d validator
+
+remote:
+	docker-compose -p ${DB_CONTAINER_NAME}  -f dc-db.yml down
+	$(shell mkdir -p ${REPACK_DIR})
+	docker-compose -p ${DB_CONTAINER_NAME}  -f dc-db.yml up -d database postgres
+
+rebuild-postgres:
+	docker-compose -p ${DB_CONTAINER_NAME} down
+	docker volume rm ${DB_CONTAINER_NAME}_fpfss_postgres_data
+	docker-compose -p ${DB_CONTAINER_NAME} -f dc-db.yml up -d
+	sleep 5
+	make migrate
+
 migrate:
 	docker run --rm -v $(shell pwd)/migrations:/migrations --network host migrate/migrate -path=/migrations/ -database "mysql://${DB_USER}:${DB_PASSWORD}@tcp(${DB_IP}:${DB_PORT})/${DB_NAME}" up
+	docker run --rm -v $(shell pwd)/postgres_migrations:/migrations --network host migrate/migrate -path=/migrations/ -database "postgres://${POSTGRES_USER}:${POSTGRES_PASSWORD}@${POSTGRES_HOST}:${POSTGRES_PORT}?sslmode=disable" up
 
 migrate-to:
 	docker run --rm -v $(shell pwd)/migrations:/migrations --network host migrate/migrate -path=/migrations/ -database "mysql://${DB_USER}:${DB_PASSWORD}@tcp(${DB_IP}:${DB_PORT})/${DB_NAME}" goto $(MIGRATION)
 
+migrate-to-pgdb:
+	docker run --rm -v $(shell pwd)/postgres_migrations:/migrations --network host migrate/migrate -path=/migrations/ -database "postgres://${POSTGRES_USER}:${POSTGRES_PASSWORD}@${POSTGRES_HOST}:${POSTGRES_PORT}?sslmode=disable" goto $(MIGRATION)
+
 validator:
 	cd .. && cd Curation-Validation-Bot && python3.9 -m uvicorn validator-server:app --host 127.0.0.1 --port 8371 --workers 8
 
@@ -30,3 +52,6 @@ restore-db:
 run:
 	export GIT_COMMIT=$(shell git rev-list -1 HEAD) && go run ./main/*.go
 
+dump-pgdb:
+	mkdir -p ./backups/pgdb/
+	docker exec -e PGPASSWORD=${POSTGRES_PASSWORD} ${POSTGRES_CONTAINER_NAME} pg_dump -U ${POSTGRES_USER} > ./backups/pgdb/pgdb-dump-${DB_NAME}-$(shell date -u +"%Y-%m-%d-%H-%M-%S").sql

config/config.go

@@ -12,6 +12,7 @@ import (
 type Config struct {
 	Port                         int64
 	OauthConf                    *oauth2.Config
+	DeviceFlowVerificaitonUrl    string
 	AuthBotToken                 string
 	FlashpointServerID           string
 	SecurecookieHashKeyPrevious  string
@@ -27,6 +28,10 @@ type Config struct {
 	DBIP                         string
 	DBPort                       int64
 	DBName                       string
+	PostgresUser                 string
+	PostgresPassword             string
+	PostgresHost                 string
+	PostgresPort                 int64
 	NotificationBotToken         string
 	NotificationChannelID        string
 	CurationFeedChannelID        string
@@ -38,6 +43,14 @@ type Config struct {
 	FixesDirFullPath             string
 	SubmissionsDirFullPath       string
 	SubmissionImagesDirFullPath  string
+	SystemUid                    int64
+	ImagesCdn                    string
+	ImagesCdnCompressed          bool
+	MinLauncherVersion           string
+	DataPacksDir                 string
+	ImagesDir                    string
+	DeletedDataPacksDir          string
+	DeletedImagesDir             string
 }
 
 func EnvString(name string) string {
@@ -88,6 +101,7 @@ func GetConfig(l *logrus.Entry) *Config {
 				AuthStyle: oauth2.AuthStyleInParams,
 			},
 		},
+		DeviceFlowVerificaitonUrl:    EnvString("DEVICE_FLOW_VERIFICATION_URL"),
 		AuthBotToken:                 EnvString("AUTH_BOT_TOKEN"),
 		FlashpointServerID:           EnvString("FLASHPOINT_SERVER_ID"),
 		SecurecookieHashKeyPrevious:  EnvString("SECURECOOKIE_HASH_KEY_PREVIOUS"),
@@ -101,6 +115,10 @@ func GetConfig(l *logrus.Entry) *Config {
 		DBIP:                         EnvString("DB_IP"),
 		DBPort:                       EnvInt("DB_PORT"),
 		DBName:                       EnvString("DB_NAME"),
+		PostgresUser:                 EnvString("POSTGRES_USER"),
+		PostgresPassword:             EnvString("POSTGRES_PASSWORD"),
+		PostgresHost:                 EnvString("POSTGRES_HOST"),
+		PostgresPort:                 EnvInt("POSTGRES_PORT"),
 		NotificationBotToken:         EnvString("NOTIFICATION_BOT_TOKEN"),
 		NotificationChannelID:        EnvString("NOTIFICATION_CHANNEL_ID"),
 		CurationFeedChannelID:        EnvString("CURATION_FEED_CHANNEL_ID"),
@@ -112,5 +130,13 @@ func GetConfig(l *logrus.Entry) *Config {
 		FixesDirFullPath:             EnvString("FIXES_DIR_FULL_PATH"),
 		SubmissionsDirFullPath:       EnvString("SUBMISSIONS_DIR_FULL_PATH"),
 		SubmissionImagesDirFullPath:  EnvString("SUBMISSION_IMAGES_DIR_FULL_PATH"),
+		SystemUid:                    EnvInt("SYSTEM_UID"),
+		ImagesCdn:                    EnvString("IMAGES_CDN"),
+		ImagesCdnCompressed:          EnvBool("IMAGES_CDN_COMPRESSED"),
+		MinLauncherVersion:           EnvString("MIN_LAUNCHER_VERSION"),
+		DataPacksDir:                 EnvString("DATA_PACKS_PATH"),
+		ImagesDir:                    EnvString("IMAGES_PATH"),
+		DeletedDataPacksDir:          EnvString("DELETED_DATA_PACKS_PATH"),
+		DeletedImagesDir:             EnvString("DELETED_IMAGES_PATH"),
 	}
 }

constants/constants.go

@@ -74,6 +74,12 @@ const (
 	ResourceKeyFixFileID             = "fix-file-id"
 	ResourceKeyUserID                = "user-id"
 	ResourceKeyTempName              = "temp-name"
+	ResourceKeyTagID                 = "tag-id"
+	ResourceKeyGameID                = "game-id"
+	ResourceKeyGameRevision          = "revision-date"
+	ResourceKeyGameDataDate          = "game-data-date"
+	ResourceKeyReason                = "reason"
+	ResourceKeyHash                  = "hash"
 )
 
 const (
@@ -100,3 +106,11 @@ const (
 	SubmissionStatusFinalizing = "finalizing"
 	SubmissionStatusSuccess    = "success"
 )
+
+func GetValidDeleteReasons() []string {
+	return []string{"Duplicate", "Owner Request", "Still On Sale", "Blacklisted Content"}
+}
+
+func GetValidRestoreReasons() []string {
+	return []string{"Wrong Delete Reason", "Taken Off Sale", "Removed From Blacklist"}
+}

constants/roles.go

@@ -10,6 +10,7 @@ const (
 	RoleMechanic      = "Mechanic"
 	RoleHunter        = "Hunter"
 	RoleTrialCurator  = "Trial Curator"
+	RoleTrialEditor   = "Trial Editor"
 	RoleTheBlue       = "The Blue"
 	RoleTheD          = "The D"
 )
@@ -35,6 +36,12 @@ func TrialCuratorRoles() []string {
 	}
 }
 
+func TrialEditorRoles() []string {
+	return []string{
+		RoleTrialEditor,
+	}
+}
+
 func DeleterRoles() []string {
 	return []string{
 		RoleAdministrator,
@@ -51,6 +58,7 @@ func DeciderRoles() []string {
 
 func AdderRoles() []string {
 	return []string{
+		RoleModerator,
 		RoleAdministrator,
 	}
 }
@@ -88,6 +96,10 @@ func IsTrialCurator(roles []string) bool {
 	return HasAnyRole(roles, TrialCuratorRoles())
 }
 
+func isTrialEditor(roles []string) bool {
+	return HasAnyRole(roles, TrialEditorRoles())
+}
+
 // IsDeleter allows users to soft delete things
 func IsDeleter(roles []string) bool {
 	return HasAnyRole(roles, DeleterRoles())
@@ -107,3 +119,7 @@ func IsAdder(roles []string) bool {
 func IsGod(roles []string) bool {
 	return HasAnyRole(roles, GodRoles())
 }
+
+func IsGodOrColin(roles []string, uid int64) bool {
+	return HasAnyRole(roles, GodRoles()) || uid == 689080719460663414
+}

database/interfaces.go

@@ -4,9 +4,60 @@ import (
 	"context"
 	"database/sql"
 	"github.com/Dri0m/flashpoint-submission-system/types"
+	"github.com/jackc/pgx/v5"
 	"time"
 )
 
+type PGDAL interface {
+	NewSession(ctx context.Context) (PGDBSession, error)
+
+	CountSinceDate(dbs PGDBSession, modifiedAfter *string) (int, error)
+
+	SearchTags(dbs PGDBSession, modifiedAfter *string) ([]*types.Tag, error)
+	SearchPlatforms(dbs PGDBSession, modifiedAfter *string) ([]*types.Platform, error)
+	SearchGames(dbs PGDBSession, modifiedAfter *string, modifiedBefore *string, broad bool, afterId *string) ([]*types.Game, []*types.AdditionalApp, []*types.GameData, [][]string, [][]string, error)
+	SearchDeletedGames(dbs PGDBSession, modifiedAfter *string) ([]*types.DeletedGame, error)
+
+	GetTagCategories(dbs PGDBSession) ([]*types.TagCategory, error)
+	GetGamesUsingTagTotal(dbs PGDBSession, tagId int64) (int64, error)
+	SaveGame(dbs PGDBSession, game *types.Game, uid int64) error
+	SaveTag(dbs PGDBSession, tag *types.Tag, uid int64) error
+	DeveloperImportDatabaseJson(dbs PGDBSession, data *types.LauncherDump) error
+
+	GetTagCategory(dbs PGDBSession, categoryId int64) (*types.TagCategory, error)
+	GetTag(dbs PGDBSession, tagId int64) (*types.Tag, error)
+	GetTagByName(dbs PGDBSession, tagName string) (*types.Tag, error)
+	GetPlatform(dbs PGDBSession, platformId int64) (*types.Platform, error)
+	GetPlatformByName(dbs PGDBSession, platformName string) (*types.Platform, error)
+	GetGame(dbs PGDBSession, gameId string) (*types.Game, error)
+	GetGameDataIndex(dbs PGDBSession, gameId string, date int64) (*types.GameDataIndex, error)
+	GetGameRevisionInfo(dbs PGDBSession, gameId string) ([]*types.RevisionInfo, error)
+	GetTagRevisionInfo(dbs PGDBSession, tagId int64) ([]*types.RevisionInfo, error)
+
+	GetMetadataStats(dbs PGDBSession) (*types.MetadataStatsPageDataBare, error)
+
+	DeleteGame(dbs PGDBSession, gameId string, uid int64, reason string, imagesPath string, gamesPath string, deletedImagesPath string, deletedGamesPath string) error
+
+	RestoreGame(dbs PGDBSession, gameId string, uid int64, reason string, imagesPath string, gamesPath string, deletedImagesPath string, deletedGamesPath string) error
+
+	GetOrCreateTagCategory(dbs PGDBSession, categoryName string) (*types.TagCategory, error)
+	GetOrCreateTag(dbs PGDBSession, tagName string, tagCategory string, reason string, uid int64) (*types.Tag, error)
+	GetOrCreatePlatform(dbs PGDBSession, platformName string, reason string, uid int64) (*types.Platform, error)
+
+	AddSubmissionFromValidator(dbs PGDBSession, uid int64, vr *types.ValidatorRepackResponse) (*types.Game, error)
+	AddGameData(dbs PGDBSession, uid int64, gameId string, vr *types.ValidatorRepackResponse) (*types.GameData, error)
+
+	IndexerGetNext(ctx context.Context) (*types.GameData, error)
+	IndexerInsert(ctx context.Context, crc32sum []byte, md5sum []byte, sha256sum []byte, sha1sum []byte,
+		size uint64, path string, gameId string, zipDate time.Time) error
+	IndexerMarkFailure(ctx context.Context, gameId string, zipDate time.Time) error
+
+	GetIndexMatchesHash(dbs PGDBSession, hashType string, hashStr string) ([]*types.IndexMatchData, error)
+
+	UpdateTagsFromTagsList(dbs PGDBSession, tagsList []types.Tag) error
+	ApplyGamePatch(dbs PGDBSession, uid int64, game *types.Game, patch *types.GameContentPatch, addApps []*types.CurationAdditionalApp) error
+}
+
 type DAL interface {
 	NewSession(ctx context.Context) (DBSession, error)
 	StoreSession(dbs DBSession, key string, uid int64, durationSeconds int64) error
@@ -90,6 +141,8 @@ type DAL interface {
 
 	GetUsers(dbs DBSession) ([]*types.User, error)
 	GetCommentsByUserIDAndAction(dbs DBSession, uid int64, action string) ([]*types.Comment, error)
+
+	PopulateRevisionInfo(dbs DBSession, revisions []*types.RevisionInfo) error
 }
 
 type DBSession interface {
@@ -98,3 +151,10 @@ type DBSession interface {
 	Tx() *sql.Tx
 	Ctx() context.Context
 }
+
+type PGDBSession interface {
+	Commit() error
+	Rollback() error
+	Tx() pgx.Tx
+	Ctx() context.Context
+}

database/mysqldal.go

@@ -320,26 +320,30 @@ func (d *mysqlDAL) GetExtendedSubmissionFilesBySubmissionID(dbs DBSession, sid i
 func (d *mysqlDAL) StoreCurationMeta(dbs DBSession, cm *types.CurationMeta) error {
 	_, err := dbs.Tx().ExecContext(dbs.Ctx(), `INSERT INTO curation_meta (fk_submission_file_id, application_path, developer, extreme, game_notes, languages,
                            launch_command, original_description, play_mode, platform, publisher, release_date, series, source, status,
-                           tags, tag_categories, title, alternate_titles, library, version, curation_notes, mount_parameters) 
-                           VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`,
+                           tags, tag_categories, title, alternate_titles, library, version, curation_notes, mount_parameters, uuid, game_exists,
+                           primary_platform) 
+                           VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`,
 		cm.SubmissionFileID, cm.ApplicationPath, cm.Developer, cm.Extreme, cm.GameNotes, cm.Languages,
 		cm.LaunchCommand, cm.OriginalDescription, cm.PlayMode, cm.Platform, cm.Publisher, cm.ReleaseDate, cm.Series, cm.Source, cm.Status,
-		cm.Tags, cm.TagCategories, cm.Title, cm.AlternateTitles, cm.Library, cm.Version, cm.CurationNotes, cm.MountParameters)
+		cm.Tags, cm.TagCategories, cm.Title, cm.AlternateTitles, cm.Library, cm.Version, cm.CurationNotes, cm.MountParameters, cm.UUID, cm.GameExists,
+		cm.PrimaryPlatform)
 	return err
 }
 
 // GetCurationMetaBySubmissionFileID returns curation meta for given submission file
 func (d *mysqlDAL) GetCurationMetaBySubmissionFileID(dbs DBSession, sfid int64) (*types.CurationMeta, error) {
 	row := dbs.Tx().QueryRowContext(dbs.Ctx(), `SELECT submission_file.fk_submission_id, application_path, developer, extreme, game_notes, languages,
                            launch_command, original_description, play_mode, platform, publisher, release_date, series, source, status,
-                           tags, tag_categories, title, alternate_titles, library, version, curation_notes, mount_parameters 
+                           tags, tag_categories, title, alternate_titles, library, version, curation_notes, mount_parameters, uuid, game_exists,
+                           primary_platform
 		FROM curation_meta JOIN submission_file ON curation_meta.fk_submission_file_id = submission_file.id
 		WHERE fk_submission_file_id=? AND submission_file.deleted_at IS NULL`, sfid)
 
 	c := &types.CurationMeta{SubmissionFileID: sfid}
 	err := row.Scan(&c.SubmissionID, &c.ApplicationPath, &c.Developer, &c.Extreme, &c.GameNotes, &c.Languages,
 		&c.LaunchCommand, &c.OriginalDescription, &c.PlayMode, &c.Platform, &c.Publisher, &c.ReleaseDate, &c.Series, &c.Source, &c.Status,
-		&c.Tags, &c.TagCategories, &c.Title, &c.AlternateTitles, &c.Library, &c.Version, &c.CurationNotes, &c.MountParameters)
+		&c.Tags, &c.TagCategories, &c.Title, &c.AlternateTitles, &c.Library, &c.Version, &c.CurationNotes, &c.MountParameters, &c.UUID, &c.GameExists,
+		&c.PrimaryPlatform)
 	if err != nil {
 		return nil, err
 	}
@@ -365,6 +369,22 @@ func (d *mysqlDAL) StoreComment(dbs DBSession, c *types.Comment) error {
 	return nil
 }
 
+func (d *mysqlDAL) PopulateRevisionInfo(dbs DBSession, revisions []*types.RevisionInfo) error {
+	for _, revision := range revisions {
+		var avatar string
+		err := dbs.Tx().QueryRowContext(dbs.Ctx(), `SELECT username, avatar
+		FROM discord_user
+		WHERE discord_user.id = ?`,
+			revision.AuthorID).
+			Scan(&revision.Username, &avatar)
+		if err != nil {
+			return err
+		}
+		revision.AvatarURL = utils.FormatAvatarURL(revision.AuthorID, avatar)
+	}
+	return nil
+}
+
 // GetExtendedCommentsBySubmissionID returns all comments with author data for a given submission
 func (d *mysqlDAL) GetExtendedCommentsBySubmissionID(dbs DBSession, sid int64) ([]*types.ExtendedComment, error) {
 	rows, err := dbs.Tx().QueryContext(dbs.Ctx(), `