more cache issues

.env

@@ -2,7 +2,7 @@
 S3_ENDPOINT=localhost:9000
 S3_ACCESS_KEY=testo
 S3_SECRET_KEY=testotesto
-S3_BUCKET=dev
 S3_DISABLE_SSL=true
 ADDRESS=:8080
-VERBOSE=true
+VERBOSE=true
+DB_CONNECTION=s3Browser:hunter2@/s3Browser

cmd/s3Browser.go

@@ -12,12 +12,12 @@ type args struct {
 	S3Endpoint   string `arg:"--s3-endpoint,required,env:S3_ENDPOINT" help:"host[:port]" placeholder:"ENDPOINT"`
 	S3AccessKey  string `arg:"--s3-access-key,required,env:S3_ACCESS_KEY" placeholder:"ACCESS_KEY"`
 	S3SecretKey  string `arg:"--s3-secret-key,required,env:S3_SECRET_KEY" placeholder:"SECRET_KEY"`
-	S3Bucket     string `arg:"--s3-bucket,required,env:S3_BUCKET" placeholder:"BUCKET"`
 	S3DisableSSL bool   `arg:"--s3-disable-ssl,env:S3_DISABLE_SSL" default:"false"`
 	Address      string `arg:"--address,env:ADDRESS" default:":3000" help:"what address to listen on" placeholder:"ADDRESS"`
 	CacheTTL     int64  `arg:"--cache-ttl,env:CACHE_TTL" help:"Time in seconds" default:"30" placeholder:"TTL"`
 	CacheCleanup int64  `arg:"--cache-cleanup,env:CACHE_CLEANUP" help:"Time in seconds" default:"60" placeholder:"CLEANUP"`
 	Verbose      bool   `arg:"-v,--verbose,env:VERBOSE" help:"verbosity level" default:"false"`
+	DBConnection string `arg:"--db,required,env:DB_CONNECTION" help:"DSN in format: https://github.com/go-sql-driver/mysql#dsn-data-source-name"`
 }
 
 func (args) Version() string {
@@ -34,7 +34,7 @@ func main() {
 		S3SSL:        !args.S3DisableSSL,
 		S3AccessKey:  args.S3AccessKey,
 		S3SecretKey:  args.S3SecretKey,
-		S3Bucket:     args.S3Bucket,
+		DSN:          args.DBConnection,
 		CacheTTL:     time.Duration(args.CacheTTL) * time.Second,
 		CacheCleanup: time.Duration(args.CacheCleanup) * time.Second,
 		Address:      args.Address,

internal/cache/cache.go

@@ -0,0 +1,9 @@
+package cache
+
+import (
+	"github.com/graph-gophers/dataloader"
+)
+
+type S3Cache interface {
+	dataloader.Cache
+}

internal/db/db.go

@@ -19,18 +19,16 @@ type DB struct {
 	dbConn *sql.DB
 }
 
-func NewDB(driver, dataSourceName string) (*DB, error) {
+func NewDB(dataSourceName string) (*DB, error) {
-	db, err := sql.Open(driver, dataSourceName)
+	db, err := sql.Open("mysql", dataSourceName)
 
 	if err != nil {
 		return nil, err
 	}
 
-	if driver == "mysql" {
+	db.SetConnMaxLifetime(time.Minute * 3)
-		db.SetConnMaxLifetime(time.Minute * 3)
+	db.SetMaxOpenConns(10)
-		db.SetMaxOpenConns(10)
+	db.SetMaxIdleConns(10)
-		db.SetMaxIdleConns(10)
-	}
 
 	return &DB{
 		dbConn: db,

internal/gql/mutations.go

@@ -11,6 +11,7 @@ import (
 	helper "git.kapelle.org/niklas/s3browser/internal/helper"
 	"git.kapelle.org/niklas/s3browser/internal/loader"
 	types "git.kapelle.org/niklas/s3browser/internal/types"
+	log "github.com/sirupsen/logrus"
 )
 
 func deleteMutation(ctx context.Context, id types.ID) error {
@@ -20,6 +21,7 @@ func deleteMutation(ctx context.Context, id types.ID) error {
 		return fmt.Errorf("Failed to get s3Client from context")
 	}
 
+	log.Debug("S3 'RemoveObject': ", id)
 	// TODO: it is posible to remove multiple objects with a single call.
 	// Is it better to batch this?
 	err := s3Client.RemoveObject(ctx, id.Bucket, id.Key, minio.RemoveObjectOptions{})
@@ -28,7 +30,7 @@ func deleteMutation(ctx context.Context, id types.ID) error {
 		return err
 	}
 
-	ctx.Value("loader").(*loader.Loader).InvalidateCacheForFile(ctx, id)
+	ctx.Value("loader").(*loader.Loader).InvalidedCacheForId(ctx, id)
 
 	return nil
 }
@@ -47,6 +49,7 @@ func copyMutation(ctx context.Context, src, dest types.ID) (*types.File, error)
 		dest.Key += helper.GetFilenameFromKey(src.Key)
 	}
 
+	log.Debug("S3 'CopyObject': ", src, "-->", dest)
 	info, err := s3Client.CopyObject(ctx, minio.CopyDestOptions{
 		Bucket: dest.Bucket,
 		Object: dest.Key,
@@ -66,7 +69,7 @@ func copyMutation(ctx context.Context, src, dest types.ID) (*types.File, error)
 
 	newID.Normalize()
 
-	ctx.Value("loader").(*loader.Loader).InvalidateCacheForFile(ctx, newID)
+	ctx.Value("loader").(*loader.Loader).InvalidedCacheForId(ctx, newID)
 
 	return &types.File{
 		ID: newID,
@@ -103,6 +106,7 @@ func moveDirMutation(ctx context.Context, src, dest types.ID) ([]*types.File, er
 		}
 		newID.Normalize()
 
+		log.Debug("S3 'CopyObject': ", src, "-->", dest)
 		_, err := s3Client.CopyObject(ctx, minio.CopyDestOptions{
 			Bucket: dest.Bucket,
 			Object: newID.Key,
@@ -117,11 +121,16 @@ func moveDirMutation(ctx context.Context, src, dest types.ID) ([]*types.File, er
 
 		deleteMutation(ctx, file.ID)
 
+		loader.InvalidedCacheForId(ctx, newID)
+		loader.InvalidedCacheForId(ctx, file.ID)
+
 		result = append(result, &types.File{
 			ID: newID,
 		})
 	}
 
+	loader.InvalidedCacheForId(ctx, src)
+
 	return result, nil
 }
 
@@ -139,6 +148,7 @@ func moveFileMutation(ctx context.Context, src, dest types.ID) (*types.File, err
 		dest.Key += helper.GetFilenameFromKey(src.Key)
 	}
 
+	log.Debug("S3 'CopyObject': ", src, "-->", dest)
 	// There is no (spoon) move. Only copy and delete
 	info, err := s3Client.CopyObject(ctx, minio.CopyDestOptions{
 		Bucket: dest.Bucket,
@@ -165,7 +175,7 @@ func moveFileMutation(ctx context.Context, src, dest types.ID) (*types.File, err
 
 	newId.Normalize()
 
-	ctx.Value("loader").(*loader.Loader).InvalidateCacheForFile(ctx, newId)
+	ctx.Value("loader").(*loader.Loader).InvalidedCacheForId(ctx, newId)
 
 	return &types.File{
 		ID: newId,
@@ -180,6 +190,7 @@ func createDirectory(ctx context.Context, id types.ID) (*types.Directory, error)
 		return nil, fmt.Errorf("Failed to get s3Client from context")
 	}
 
+	log.Debug("S3 'PutObject': ", id)
 	info, err := s3Client.PutObject(ctx, id.Bucket, id.Key, strings.NewReader(""), 0, minio.PutObjectOptions{
 		ContentType: "application/x-directory",
 	})
@@ -195,7 +206,7 @@ func createDirectory(ctx context.Context, id types.ID) (*types.Directory, error)
 
 	newID.Normalize()
 
-	ctx.Value("loader").(*loader.Loader).InvalidateCacheForDir(ctx, newID)
+	ctx.Value("loader").(*loader.Loader).InvalidedCacheForId(ctx, newID)
 
 	return &types.Directory{
 		ID: newID,
@@ -241,13 +252,14 @@ func deleteDirectory(ctx context.Context, id types.ID) error {
 	// This is at least the behavior when working with minio as s3 backend
 	// TODO: check if this is normal behavior when working with s3
 	if len(files) == 0 {
+		log.Debug("S3 'RemoveObject': ", id)
 		err := s3Client.RemoveObject(ctx, id.Bucket, id.Key, minio.RemoveObjectOptions{})
 		if err != nil {
 			return err
 		}
 	}
 
-	loader.InvalidateCacheForDir(ctx, id)
+	loader.InvalidedCacheForId(ctx, id)
 
 	return nil
 }

internal/helper/helper.go

@@ -17,7 +17,6 @@ func GetFilenameFromKey(id string) string {
 }
 
 func DeleteMultiple(ctx context.Context, s3Client minio.Client, bucket string, keys []string) error {
-	log.Debug("Remove multiple objects")
 	objectsCh := make(chan minio.ObjectInfo, 1)
 
 	go func() {
@@ -29,6 +28,7 @@ func DeleteMultiple(ctx context.Context, s3Client minio.Client, bucket string, k
 		}
 	}()
 
+	log.Debug("S3 'RemoveObject': ", keys)
 	for err := range s3Client.RemoveObjects(ctx, bucket, objectsCh, minio.RemoveObjectsOptions{}) {
 		log.Error("Failed to delete object ", err.ObjectName, " because: ", err.Err.Error())
 		// TODO: error handel

internal/httpserver/httpServer.go

@@ -123,7 +123,7 @@ func httpGetFile(ctx context.Context, rw http.ResponseWriter, r *http.Request) {
 		return
 	}
 
-	log.Debug("S3 call 'StatObject': ", id)
+	log.Debug("S3 'StatObject': ", id)
 	objInfo, err := s3Client.StatObject(context.Background(), id.Bucket, id.Key, minio.GetObjectOptions{})
 
 	if err != nil {
@@ -138,7 +138,7 @@ func httpGetFile(ctx context.Context, rw http.ResponseWriter, r *http.Request) {
 		return
 	}
 
-	log.Debug("S3 call 'GetObject': ", id)
+	log.Debug("S3 'GetObject': ", id)
 	obj, err := s3Client.GetObject(context.Background(), id.Bucket, id.Key, minio.GetObjectOptions{})
 
 	if err != nil {
@@ -180,12 +180,10 @@ func httpPostFile(ctx context.Context, rw http.ResponseWriter, r *http.Request)
 
 	id.Normalize()
 
-	log.Debug("Upload file: ", id)
-
 	contentType := r.Header.Get("Content-Type")
 	mimeType, _, _ := mime.ParseMediaType(contentType)
 
-	log.Debug("S3 call 'PutObject': ", id)
+	log.Debug("S3 'PutObject': ", id)
 	_, err := s3Client.PutObject(context.Background(), id.Bucket, id.Key, r.Body, r.ContentLength, minio.PutObjectOptions{
 		ContentType: mimeType,
 	})
@@ -196,8 +194,7 @@ func httpPostFile(ctx context.Context, rw http.ResponseWriter, r *http.Request)
 	}
 
 	loader := ctx.Value("loader").(*loader.Loader)
-	loader.InvalidateCacheForFile(ctx, *id)
+	loader.InvalidedCacheForId(ctx, *id)
-	loader.InvalidateCacheForDir(ctx, *id.Parent())
 
 	rw.WriteHeader(http.StatusCreated)
 }

internal/loader/batch.go

@@ -12,7 +12,6 @@ import (
 
 // listObjectsBatch batch func for calling s3.ListObjects()
 func listObjectsBatch(c context.Context, k dataloader.Keys) []*dataloader.Result {
-	log.Debug("listObjectsBatch: ", k.Keys())
 	var results []*dataloader.Result
 
 	s3Client, ok := c.Value("s3Client").(*minio.Client)
@@ -34,7 +33,6 @@ func listObjectsBatch(c context.Context, k dataloader.Keys) []*dataloader.Result
 
 // listObjectsRecursiveBatch just like listObjectsBatch but with recursive set to true
 func listObjectsRecursiveBatch(c context.Context, k dataloader.Keys) []*dataloader.Result {
-	log.Debug("listObjectsRecursiveBatch: ", k.Keys())
 	var results []*dataloader.Result
 
 	s3Client, ok := c.Value("s3Client").(*minio.Client)
@@ -56,7 +54,7 @@ func listObjectsRecursiveBatch(c context.Context, k dataloader.Keys) []*dataload
 
 // listObjects helper func for listObjectsBatch
 func listObjects(s3Client *minio.Client, id types.ID, recursive bool) []minio.ObjectInfo {
-	log.Debug("S3 call 'ListObjects': ", id)
+	log.Debug("S3 'ListObjects': ", id)
 	objectCh := s3Client.ListObjects(context.Background(), id.Bucket, minio.ListObjectsOptions{
 		Prefix:    id.Key,
 		Recursive: recursive,
@@ -71,7 +69,6 @@ func listObjects(s3Client *minio.Client, id types.ID, recursive bool) []minio.Ob
 }
 
 func listBucketsBatch(c context.Context, k dataloader.Keys) []*dataloader.Result {
-	log.Debug("listBucketsBatch")
 	var results []*dataloader.Result
 
 	s3Client, ok := c.Value("s3Client").(*minio.Client)
@@ -80,6 +77,7 @@ func listBucketsBatch(c context.Context, k dataloader.Keys) []*dataloader.Result
 		return handleLoaderError(k, fmt.Errorf("Failed to get s3Client from context"))
 	}
 
+	log.Debug("S3 'ListBuckets'")
 	buckets, err := s3Client.ListBuckets(c)
 
 	if err != nil {
@@ -110,6 +108,7 @@ func statObjectBatch(ctx context.Context, k dataloader.Keys) []*dataloader.Resul
 
 	for _, v := range k {
 		id := v.Raw().(types.ID)
+		log.Debug("S3 'StatObject': ", id)
 		stat, err := s3Client.StatObject(ctx, id.Bucket, id.Key, minio.GetObjectOptions{})
 		results = append(results, &dataloader.Result{
 			Data:  stat,

internal/loader/loader.go

@@ -6,11 +6,11 @@ import (
 	"path/filepath"
 	"strings"
 
+	"git.kapelle.org/niklas/s3browser/internal/cache"
 	"git.kapelle.org/niklas/s3browser/internal/helper"
 	types "git.kapelle.org/niklas/s3browser/internal/types"
 	"github.com/graph-gophers/dataloader"
 	"github.com/minio/minio-go/v7"
-	log "github.com/sirupsen/logrus"
 )
 
 type Loader struct {
@@ -18,26 +18,43 @@ type Loader struct {
 	listObjectsRecursiveLoader *dataloader.Loader
 	statObjectLoader           *dataloader.Loader
 	listBucketsLoader          *dataloader.Loader
+
+	listObjectsLoaderCache          cache.S3Cache
+	listObjectsRecursiveLoaderCache cache.S3Cache
+	statObjectLoaderCache           cache.S3Cache
+	listBucketsLoaderCache          cache.S3Cache
 }
 
 func NewLoader(config types.AppConfig) *Loader {
+	listObjectsLoaderCache := &dataloader.NoCache{}
+	listObjectsRecursiveLoaderCache := &dataloader.NoCache{}
+	statObjectLoaderCache := cache.NewTTLCache(config.CacheTTL, config.CacheCleanup)
+	listBucketsLoaderCache := cache.NewTTLCache(config.CacheTTL, config.CacheCleanup)
+
 	return &Loader{
 		listObjectsLoader: dataloader.NewBatchedLoader(
 			listObjectsBatch,
-			dataloader.WithCache(&dataloader.NoCache{}),
+			dataloader.WithCache(listObjectsLoaderCache),
 		),
+		listObjectsLoaderCache: listObjectsLoaderCache,
+
 		listObjectsRecursiveLoader: dataloader.NewBatchedLoader(
 			listObjectsRecursiveBatch,
-			dataloader.WithCache(&dataloader.NoCache{}),
+			dataloader.WithCache(listObjectsRecursiveLoaderCache),
 		),
+		listObjectsRecursiveLoaderCache: listObjectsRecursiveLoaderCache,
+
 		statObjectLoader: dataloader.NewBatchedLoader(
 			statObjectBatch,
-			dataloader.WithCache(&dataloader.NoCache{}),
+			dataloader.WithCache(statObjectLoaderCache),
 		),
+		statObjectLoaderCache: statObjectLoaderCache,
+
 		listBucketsLoader: dataloader.NewBatchedLoader(
 			listBucketsBatch,
-			dataloader.WithCache(&dataloader.NoCache{}),
+			dataloader.WithCache(listBucketsLoaderCache),
 		),
+		listBucketsLoaderCache: listBucketsLoaderCache,
 	}
 }
 
@@ -144,18 +161,17 @@ func (l *Loader) GetFilesRecursive(ctx context.Context, path types.ID) ([]types.
 	return files, nil
 }
 
-func (l *Loader) InvalidateCacheForFile(ctx context.Context, id types.ID) {
+func (l *Loader) InvalidedCacheForId(ctx context.Context, id types.ID) {
-	log.Debug("Clear cache for file: ", id.String())
 	parent := id.Parent()
 
 	l.statObjectLoader.Clear(ctx, id)
+
+	// Code below is useless for now until we use a propper cache for "listObjectsLoader" and "listObjectsRecursiveLoader"
+	// TODO: implement cache invalidation for "listObjectsLoader" and "listObjectsRecursiveLoader"
 	l.listObjectsLoader.Clear(ctx, id).Clear(ctx, parent)
-}
 
-func (l *Loader) InvalidateCacheForDir(ctx context.Context, path types.ID) {
+	// Remove up from recursive list
-	log.Debug("Clear cache for dir: ", path.String())
+	for rParent := parent; rParent != nil; rParent = rParent.Parent() {
-	parent := helper.GetParentDir(path)
+		l.listObjectsRecursiveLoader.Clear(ctx, rParent)
-
+	}
-	l.listObjectsLoader.Clear(ctx, path).Clear(ctx, parent)
-	l.listObjectsRecursiveLoader.Clear(ctx, path).Clear(ctx, parent)
 }

internal/s3Broswer.go

@@ -2,7 +2,6 @@ package s3browser
 
 import (
 	"context"
-	"fmt"
 
 	"github.com/minio/minio-go/v7"
 	"github.com/minio/minio-go/v7/pkg/credentials"
@@ -17,26 +16,10 @@ import (
 
 // setupS3Client connect the s3Client
 func setupS3Client(config types.AppConfig) (*minio.Client, error) {
-	minioClient, err := minio.New(config.S3Endoint, &minio.Options{
+	return minio.New(config.S3Endoint, &minio.Options{
 		Creds:  credentials.NewStaticV4(config.S3AccessKey, config.S3SecretKey, ""),
 		Secure: config.S3SSL,
 	})
-
-	if err != nil {
-		return nil, err
-	}
-
-	exists, err := minioClient.BucketExists(context.Background(), config.S3Bucket)
-
-	if err != nil {
-		return nil, err
-	}
-
-	if !exists {
-		return nil, fmt.Errorf("Bucket '%s' does not exist", config.S3Bucket)
-	}
-
-	return minioClient, nil
 }
 
 // Start starts the app
@@ -47,24 +30,20 @@ func Start(config types.AppConfig) {
 	}
 
 	log.Info("Starting")
-	log.Debug("Setting up s3 client")
 	s3Client, err := setupS3Client(config)
 
 	if err != nil {
 		log.Error("Failed to setup s3 client: ", err.Error())
 		return
 	}
-	log.Info("s3 client connected")
 
-	dbStore, err := db.NewDB("mysql", "s3Browser:hunter2@/s3Browser")
+	dbStore, err := db.NewDB(config.DSN)
 	if err != nil {
 		log.Error("Failed to connect DB: ", err.Error())
 	}
 
-	log.Debug("Creating dataloader")
 	loader := loader.NewLoader(config)
 
-	log.Debug("Generating graphq schema")
 	gql.GraphqlTypes()
 	schema, err := gql.GraphqlSchema()
 
@@ -77,7 +56,6 @@ func Start(config types.AppConfig) {
 	resolveContext = context.WithValue(resolveContext, "loader", loader)
 	resolveContext = context.WithValue(resolveContext, "dbStore", dbStore)
 
-	log.Debug("Starting HTTP server")
 	err = httpserver.InitHttp(resolveContext, schema, config.Address)
 
 	if err != nil {

internal/types/types.go

@@ -12,7 +12,7 @@ type AppConfig struct {
 	S3AccessKey  string
 	S3SecretKey  string
 	S3SSL        bool
-	S3Bucket     string
+	DSN          string
 	CacheTTL     time.Duration
 	CacheCleanup time.Duration
 	Address      string