fix get value db

.env

@@ -2,7 +2,7 @@
 S3_ENDPOINT=localhost:9000
 S3_ACCESS_KEY=testo
 S3_SECRET_KEY=testotesto
-S3_BUCKET=dev
 S3_DISABLE_SSL=true
 ADDRESS=:8080
 VERBOSE=true
+DB_CONNECTION=s3Browser:hunter2@/s3Browser

cmd/s3Browser.go

@@ -12,12 +12,12 @@ type args struct {
 	S3Endpoint   string `arg:"--s3-endpoint,required,env:S3_ENDPOINT" help:"host[:port]" placeholder:"ENDPOINT"`
 	S3AccessKey  string `arg:"--s3-access-key,required,env:S3_ACCESS_KEY" placeholder:"ACCESS_KEY"`
 	S3SecretKey  string `arg:"--s3-secret-key,required,env:S3_SECRET_KEY" placeholder:"SECRET_KEY"`
-	S3Bucket     string `arg:"--s3-bucket,required,env:S3_BUCKET" placeholder:"BUCKET"`
 	S3DisableSSL bool   `arg:"--s3-disable-ssl,env:S3_DISABLE_SSL" default:"false"`
 	Address      string `arg:"--address,env:ADDRESS" default:":3000" help:"what address to listen on" placeholder:"ADDRESS"`
 	CacheTTL     int64  `arg:"--cache-ttl,env:CACHE_TTL" help:"Time in seconds" default:"30" placeholder:"TTL"`
 	CacheCleanup int64  `arg:"--cache-cleanup,env:CACHE_CLEANUP" help:"Time in seconds" default:"60" placeholder:"CLEANUP"`
 	Verbose      bool   `arg:"-v,--verbose,env:VERBOSE" help:"verbosity level" default:"false"`
+	DBConnection string `arg:"--db,required,env:DB_CONNECTION" help:"DSN in format: https://github.com/go-sql-driver/mysql#dsn-data-source-name"`
 }
 
 func (args) Version() string {
@@ -34,7 +34,7 @@ func main() {
 		S3SSL:        !args.S3DisableSSL,
 		S3AccessKey:  args.S3AccessKey,
 		S3SecretKey:  args.S3SecretKey,
-		S3Bucket:     args.S3Bucket,
+		DSN:          args.DBConnection,
 		CacheTTL:     time.Duration(args.CacheTTL) * time.Second,
 		CacheCleanup: time.Duration(args.CacheCleanup) * time.Second,
 		Address:      args.Address,

go.mod

@@ -14,5 +14,6 @@ require (
 	github.com/opentracing/opentracing-go v1.2.0 // indirect
 	github.com/patrickmn/go-cache v2.1.0+incompatible
 	github.com/sirupsen/logrus v1.8.1
+	github.com/stretchr/testify v1.7.0
 	golang.org/x/crypto v0.0.0-20201216223049-8b5274cf687f
 )

go.sum

@@ -67,6 +67,8 @@ github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXf
 github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
 github.com/stretchr/testify v1.4.0 h1:2E4SXV/wtOkTonXsotYi4li6zVWxYlZuYNCXe9XRJyk=
 github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
+github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5CcY=
+github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
 golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
 golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
 golang.org/x/crypto v0.0.0-20201216223049-8b5274cf687f h1:aZp0e2vLN4MToVqnjNEYEtrEA8RH8U8FN1CU7JgqsPU=
@@ -94,3 +96,5 @@ gopkg.in/ini.v1 v1.57.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k=
 gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 gopkg.in/yaml.v2 v2.2.8 h1:obN1ZagJSUGI0Ek/LBmuj4SNLPfIny3KsKFopxRdj10=
 gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
+gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c h1:dUUwHk2QECo/6vqA44rthZ8ie2QXMNeKRTHCNY2nXvo=
+gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=

internal/cache/cache.go

@@ -0,0 +1,9 @@
+package cache
+
+import (
+	"github.com/graph-gophers/dataloader"
+)
+
+type S3Cache interface {
+	dataloader.Cache
+}

internal/db/db.go

@@ -1,96 +1,9 @@
 package db
 
-import (
+import "context"
-	"context"
-	"database/sql"
-	_ "embed"
-	"time"
 
-	_ "github.com/go-sql-driver/mysql"
+type DB interface {
-	"golang.org/x/crypto/bcrypt"
+	Setup() error
-)
+	CheckLogin(ctx context.Context, username, password string) (bool, error)
-
+	AddUser(ctx context.Context, username, password string) error
-//go:embed setup.sql
-var setupSql string
-
-const DB_NAME = "s3Browser"
-
-type DB struct {
-	dbConn *sql.DB
-}
-
-func NewDB(driver, dataSourceName string) (*DB, error) {
-	db, err := sql.Open(driver, dataSourceName)
-
-	if err != nil {
-		return nil, err
-	}
-
-	if driver == "mysql" {
-		db.SetConnMaxLifetime(time.Minute * 3)
-		db.SetMaxOpenConns(10)
-		db.SetMaxIdleConns(10)
-	}
-
-	return &DB{
-		dbConn: db,
-	}, nil
-}
-
-func (d *DB) Setup() error {
-	tx, err := d.dbConn.Begin()
-	if err != nil {
-		return err
-	}
-
-	_, err = tx.Exec(setupSql)
-	if err != nil {
-		tx.Rollback()
-		return err
-	}
-
-	err = tx.Commit()
-	if err != nil {
-		return err
-	}
-
-	return nil
-}
-
-func (d *DB) CheckLogin(ctx context.Context, username, password string) (bool, error) {
-	rows, err := d.dbConn.QueryContext(ctx, "SELECT password FROM user WHERE username = ?", username)
-	if err != nil {
-		return false, err
-	}
-
-	if !rows.Next() {
-		return false, nil
-	}
-
-	var passwordHash []byte
-	err = rows.Scan(&passwordHash)
-	if err != nil {
-		return false, err
-	}
-
-	if bcrypt.CompareHashAndPassword(passwordHash, []byte(password)) != nil {
-		return false, nil
-	}
-
-	return true, nil
-}
-
-func (d *DB) AddUser(ctx context.Context, username, password string) error {
-	hash, err := bcrypt.GenerateFromPassword([]byte(password), bcrypt.DefaultCost)
-	if err != nil {
-		return err
-	}
-
-	_, err = d.dbConn.ExecContext(ctx, "INSERT INTO user (username,password) VALUES (?,?)", username, hash)
-
-	if err != nil {
-		return err
-	}
-
-	return nil
 }

internal/db/mysql.go

@@ -0,0 +1,94 @@
+package db
+
+import (
+	"context"
+	"database/sql"
+	_ "embed"
+	"time"
+
+	_ "github.com/go-sql-driver/mysql"
+	"golang.org/x/crypto/bcrypt"
+)
+
+//go:embed setup.sql
+var setupSql string
+
+const DB_NAME = "s3Browser"
+
+type mysqlDB struct {
+	dbConn *sql.DB
+}
+
+func NewDB(dataSourceName string) (DB, error) {
+	db, err := sql.Open("mysql", dataSourceName)
+
+	if err != nil {
+		return nil, err
+	}
+
+	db.SetConnMaxLifetime(time.Minute * 3)
+	db.SetMaxOpenConns(10)
+	db.SetMaxIdleConns(10)
+
+	return &mysqlDB{
+		dbConn: db,
+	}, nil
+}
+
+func (d *mysqlDB) Setup() error {
+	tx, err := d.dbConn.Begin()
+	if err != nil {
+		return err
+	}
+
+	_, err = tx.Exec(setupSql)
+	if err != nil {
+		tx.Rollback()
+		return err
+	}
+
+	err = tx.Commit()
+	if err != nil {
+		return err
+	}
+
+	return nil
+}
+
+func (d *mysqlDB) CheckLogin(ctx context.Context, username, password string) (bool, error) {
+	rows, err := d.dbConn.QueryContext(ctx, "SELECT password FROM user WHERE username = ?", username)
+	if err != nil {
+		return false, err
+	}
+
+	if !rows.Next() {
+		return false, nil
+	}
+
+	var passwordHash []byte
+	err = rows.Scan(&passwordHash)
+	if err != nil {
+		return false, err
+	}
+
+	if bcrypt.CompareHashAndPassword(passwordHash, []byte(password)) != nil {
+		return false, nil
+	}
+
+	return true, nil
+}
+
+func (d *mysqlDB) AddUser(ctx context.Context, username, password string) error {
+	hash, err := bcrypt.GenerateFromPassword([]byte(password), bcrypt.DefaultCost)
+	if err != nil {
+		return err
+	}
+
+	_, err = d.dbConn.ExecContext(ctx, "INSERT INTO user (username,password) VALUES (?,?)", username, hash)
+
+	if err != nil {
+		return err
+	}
+
+	return nil
+}

internal/gql/gql_test.go

@@ -0,0 +1,75 @@
+package gql_test
+
+import (
+	"context"
+	"testing"
+
+	"git.kapelle.org/niklas/s3browser/internal/gql"
+	"git.kapelle.org/niklas/s3browser/internal/loader"
+	"git.kapelle.org/niklas/s3browser/internal/s3"
+	"github.com/graph-gophers/dataloader"
+	"github.com/graphql-go/graphql"
+	"github.com/stretchr/testify/assert"
+)
+
+func setup(t *testing.T) (*assert.Assertions, context.Context, graphql.Schema) {
+	assert := assert.New(t)
+	ctx := context.Background()
+
+	schema, _ := gql.GraphqlSchema()
+
+	s3, err := s3.NewMockS3([]string{"bucket1"})
+	assert.NoError(err)
+	ctx = context.WithValue(ctx, "s3Client", s3)
+
+	loader := loader.NewLoader(loader.CacheConfig{
+		ListObjectsLoaderCache:          &dataloader.NoCache{},
+		ListObjectsRecursiveLoaderCache: &dataloader.NoCache{},
+		StatObjectLoaderCache:           &dataloader.NoCache{},
+		ListBucketsLoaderCache:          &dataloader.NoCache{},
+	})
+	assert.NotNil(loader)
+	ctx = context.WithValue(ctx, "loader", loader)
+
+	return assert, ctx, schema
+}
+
+func do(ctx context.Context, schema graphql.Schema, query string) *graphql.Result {
+	params := graphql.Params{
+		Schema:        schema,
+		RequestString: query,
+		Context:       ctx,
+	}
+	r := graphql.Do(params)
+	return r
+}
+
+func TestCreateSchema(t *testing.T) {
+	assert := assert.New(t)
+
+	assert.NotPanics(func() {
+		gql.GraphqlTypes()
+	})
+
+	var schema graphql.Schema
+	var err error
+
+	assert.NotPanics(func() {
+		schema, err = gql.GraphqlSchema()
+	})
+
+	assert.NoError(err)
+	assert.NotNil(schema)
+}
+
+func TestAuth(t *testing.T) {
+	assert, ctx, schema := setup(t)
+
+	r := do(ctx, schema, `
+		{
+			authorized
+		}
+	`)
+	t.Logf("Data: %v", r.Data)
+	assert.Len(r.Errors, 0)
+}

internal/gql/graphqlTypes.go

@@ -13,6 +13,7 @@ import (
 	types "git.kapelle.org/niklas/s3browser/internal/types"
 )
 
+var typesInit bool = false
 var graphqlDirType *graphql.Object
 var graphqlFileType *graphql.Object
 var graphqlLoginResultType *graphql.Object
@@ -246,6 +247,8 @@ func GraphqlTypes() {
 		},
 	})
 
+	typesInit = true
+
 }
 
 //loadFile helper func for using the dataloader to get a file

internal/gql/mutations.go

@@ -5,36 +5,37 @@ import (
 	"fmt"
 	"strings"
 
-	"github.com/minio/minio-go/v7"
-
 	"git.kapelle.org/niklas/s3browser/internal/db"
 	helper "git.kapelle.org/niklas/s3browser/internal/helper"
 	"git.kapelle.org/niklas/s3browser/internal/loader"
+	"git.kapelle.org/niklas/s3browser/internal/s3"
 	types "git.kapelle.org/niklas/s3browser/internal/types"
+	log "github.com/sirupsen/logrus"
 )
 
 func deleteMutation(ctx context.Context, id types.ID) error {
-	s3Client, ok := ctx.Value("s3Client").(*minio.Client)
+	s3Client, ok := ctx.Value("s3Client").(s3.S3Service)
 
 	if !ok {
 		return fmt.Errorf("Failed to get s3Client from context")
 	}
 
+	log.Debug("S3 'RemoveObject': ", id)
 	// TODO: it is posible to remove multiple objects with a single call.
 	// Is it better to batch this?
-	err := s3Client.RemoveObject(ctx, id.Bucket, id.Key, minio.RemoveObjectOptions{})
+	err := s3Client.RemoveObject(ctx, id)
 
 	if err != nil {
 		return err
 	}
 
-	ctx.Value("loader").(*loader.Loader).InvalidateCacheForFile(ctx, id)
+	ctx.Value("loader").(*loader.Loader).InvalidedCacheForId(ctx, id)
 
 	return nil
 }
 
 func copyMutation(ctx context.Context, src, dest types.ID) (*types.File, error) {
-	s3Client, ok := ctx.Value("s3Client").(*minio.Client)
+	s3Client, ok := ctx.Value("s3Client").(s3.S3Service)
 
 	if !ok {
 		return nil, fmt.Errorf("Failed to get s3Client from context")
@@ -47,35 +48,23 @@ func copyMutation(ctx context.Context, src, dest types.ID) (*types.File, error)
 		dest.Key += helper.GetFilenameFromKey(src.Key)
 	}
 
-	info, err := s3Client.CopyObject(ctx, minio.CopyDestOptions{
+	log.Debug("S3 'CopyObject': ", src, "-->", dest)
-		Bucket: dest.Bucket,
+	err := s3Client.CopyObject(ctx, src, dest)
-		Object: dest.Key,
-	}, minio.CopySrcOptions{
-		Bucket: src.Bucket,
-		Object: src.Key,
-	})
 
 	if err != nil {
 		return nil, err
 	}
 
-	newID := types.ID{
+	ctx.Value("loader").(*loader.Loader).InvalidedCacheForId(ctx, dest)
-		Bucket: info.Bucket,
-		Key:    info.Key,
-	}
-
-	newID.Normalize()
-
-	ctx.Value("loader").(*loader.Loader).InvalidateCacheForFile(ctx, newID)
 
 	return &types.File{
-		ID: newID,
+		ID: dest,
 	}, nil
 
 }
 
 func moveDirMutation(ctx context.Context, src, dest types.ID) ([]*types.File, error) {
-	s3Client, ok := ctx.Value("s3Client").(*minio.Client)
+	s3Client, ok := ctx.Value("s3Client").(s3.S3Service)
 
 	if !ok {
 		return nil, fmt.Errorf("Failed to get s3Client from context")
@@ -103,13 +92,8 @@ func moveDirMutation(ctx context.Context, src, dest types.ID) ([]*types.File, er
 		}
 		newID.Normalize()
 
-		_, err := s3Client.CopyObject(ctx, minio.CopyDestOptions{
+		log.Debug("S3 'CopyObject': ", src, "-->", dest)
-			Bucket: dest.Bucket,
+		err := s3Client.CopyObject(ctx, file.ID, dest)
-			Object: newID.Key,
-		}, minio.CopySrcOptions{
-			Bucket: file.ID.Bucket,
-			Object: file.ID.Key,
-		})
 
 		if err != nil {
 			// TODO: handle error
@@ -117,16 +101,21 @@ func moveDirMutation(ctx context.Context, src, dest types.ID) ([]*types.File, er
 
 		deleteMutation(ctx, file.ID)
 
+		loader.InvalidedCacheForId(ctx, newID)
+		loader.InvalidedCacheForId(ctx, file.ID)
+
 		result = append(result, &types.File{
 			ID: newID,
 		})
 	}
 
+	loader.InvalidedCacheForId(ctx, src)
+
 	return result, nil
 }
 
 func moveFileMutation(ctx context.Context, src, dest types.ID) (*types.File, error) {
-	s3Client, ok := ctx.Value("s3Client").(*minio.Client)
+	s3Client, ok := ctx.Value("s3Client").(s3.S3Service)
 
 	if !ok {
 		return nil, fmt.Errorf("Failed to get s3Client from context")
@@ -139,14 +128,9 @@ func moveFileMutation(ctx context.Context, src, dest types.ID) (*types.File, err
 		dest.Key += helper.GetFilenameFromKey(src.Key)
 	}
 
+	log.Debug("S3 'CopyObject': ", src, "-->", dest)
 	// There is no (spoon) move. Only copy and delete
-	info, err := s3Client.CopyObject(ctx, minio.CopyDestOptions{
+	err := s3Client.CopyObject(ctx, src, dest)
-		Bucket: dest.Bucket,
-		Object: dest.Key,
-	}, minio.CopySrcOptions{
-		Bucket: src.Bucket,
-		Object: src.Key,
-	})
 
 	if err != nil {
 		return nil, err
@@ -158,53 +142,37 @@ func moveFileMutation(ctx context.Context, src, dest types.ID) (*types.File, err
 		return nil, err
 	}
 
-	newId := types.ID{
+	ctx.Value("loader").(*loader.Loader).InvalidedCacheForId(ctx, dest)
-		Bucket: info.Bucket,
-		Key:    info.Key,
-	}
-
-	newId.Normalize()
-
-	ctx.Value("loader").(*loader.Loader).InvalidateCacheForFile(ctx, newId)
 
 	return &types.File{
-		ID: newId,
+		ID: dest,
 	}, nil
 
 }
 
 func createDirectory(ctx context.Context, id types.ID) (*types.Directory, error) {
-	s3Client, ok := ctx.Value("s3Client").(*minio.Client)
+	s3Client, ok := ctx.Value("s3Client").(s3.S3Service)
 
 	if !ok {
 		return nil, fmt.Errorf("Failed to get s3Client from context")
 	}
 
-	info, err := s3Client.PutObject(ctx, id.Bucket, id.Key, strings.NewReader(""), 0, minio.PutObjectOptions{
+	log.Debug("S3 'PutObject': ", id)
-		ContentType: "application/x-directory",
+	err := s3Client.PutObject(ctx, id, strings.NewReader(""), 0) // TODO: s3client interface needs content type parameter
-	})
-
 	if err != nil {
 		return nil, err
 	}
 
-	newID := types.ID{
+	ctx.Value("loader").(*loader.Loader).InvalidedCacheForId(ctx, id)
-		Bucket: info.Bucket,
-		Key:    info.Key,
-	}
-
-	newID.Normalize()
-
-	ctx.Value("loader").(*loader.Loader).InvalidateCacheForDir(ctx, newID)
 
 	return &types.Directory{
-		ID: newID,
+		ID: id,
 	}, nil
 
 }
 
 func deleteDirectory(ctx context.Context, id types.ID) error {
-	s3Client, ok := ctx.Value("s3Client").(*minio.Client)
+	s3Client, ok := ctx.Value("s3Client").(s3.S3Service)
 
 	if !ok {
 		return fmt.Errorf("Failed to get s3Client from context")
@@ -225,11 +193,9 @@ func deleteDirectory(ctx context.Context, id types.ID) error {
 	}
 
 	// Delete all child files
-	var keysToDel []string
 	for _, file := range files {
-		keysToDel = append(keysToDel, file.ID.Key)
+		s3Client.RemoveObject(ctx, file.ID)
 	}
-	err = helper.DeleteMultiple(ctx, *s3Client, id.Bucket, keysToDel)
 
 	if err != nil {
 		return err
@@ -241,13 +207,14 @@ func deleteDirectory(ctx context.Context, id types.ID) error {
 	// This is at least the behavior when working with minio as s3 backend
 	// TODO: check if this is normal behavior when working with s3
 	if len(files) == 0 {
-		err := s3Client.RemoveObject(ctx, id.Bucket, id.Key, minio.RemoveObjectOptions{})
+		log.Debug("S3 'RemoveObject': ", id)
+		err := s3Client.RemoveObject(ctx, id)
 		if err != nil {
 			return err
 		}
 	}
 
-	loader.InvalidateCacheForDir(ctx, id)
+	loader.InvalidedCacheForId(ctx, id)
 
 	return nil
 }
@@ -255,7 +222,7 @@ func deleteDirectory(ctx context.Context, id types.ID) error {
 //login Checks for valid username password combination. Returns singed jwt string
 func login(ctx context.Context, username, password string) (types.LoginResult, error) {
 
-	dbStore := ctx.Value("dbStore").(*db.DB)
+	dbStore := ctx.Value("dbStore").(db.DB)
 
 	succes, err := dbStore.CheckLogin(ctx, username, password)
 

internal/gql/schema.go

@@ -15,6 +15,10 @@ import (
 //GraphqlSchema generate the schema with its root query and mutation
 func GraphqlSchema() (graphql.Schema, error) {
 
+	if !typesInit {
+		GraphqlTypes()
+	}
+
 	queryFields := graphql.Fields{
 		"files": &graphql.Field{
 			Type: graphql.NewNonNull(graphql.NewList(graphql.NewNonNull(graphqlFileType))),

internal/helper/helper.go

@@ -6,37 +6,15 @@ import (
 	"strings"
 	"time"
 
+	"git.kapelle.org/niklas/s3browser/internal/s3"
 	types "git.kapelle.org/niklas/s3browser/internal/types"
 	"github.com/golang-jwt/jwt"
-	"github.com/minio/minio-go/v7"
-	log "github.com/sirupsen/logrus"
 )
 
 func GetFilenameFromKey(id string) string {
 	return filepath.Base(id)
 }
 
-func DeleteMultiple(ctx context.Context, s3Client minio.Client, bucket string, keys []string) error {
-	log.Debug("Remove multiple objects")
-	objectsCh := make(chan minio.ObjectInfo, 1)
-
-	go func() {
-		defer close(objectsCh)
-		for _, id := range keys {
-			objectsCh <- minio.ObjectInfo{
-				Key: id,
-			}
-		}
-	}()
-
-	for err := range s3Client.RemoveObjects(ctx, bucket, objectsCh, minio.RemoveObjectsOptions{}) {
-		log.Error("Failed to delete object ", err.ObjectName, " because: ", err.Err.Error())
-		// TODO: error handel
-	}
-
-	return nil
-}
-
 func GetParentDir(id types.ID) types.ID {
 	dirs := strings.Split(id.Key, "/")
 
@@ -57,17 +35,10 @@ func GetParentDir(id types.ID) types.ID {
 	return parent
 }
 
-func ObjInfoToFile(objInfo minio.ObjectInfo, bucket string) *types.File {
+func ObjInfoToFile(objInfo s3.Object, bucket string) *types.File {
-	objID := types.ID{
-		Bucket: bucket,
-		Key:    objInfo.Key,
-	}
-
-	objID.Normalize()
-
 	return &types.File{
-		ID:           objID,
+		ID:           objInfo.ID,
-		Name:         GetFilenameFromKey(objID.Key),
+		Name:         objInfo.ID.Name(),
 		Size:         objInfo.Size,
 		ContentType:  objInfo.ContentType,
 		ETag:         objInfo.ETag,

internal/httpserver/httpServer.go

@@ -4,9 +4,7 @@ import (
 	"context"
 	"fmt"
 	"io"
-	"mime"
 	"net/http"
-	"path/filepath"
 	"time"
 
 	"github.com/golang-jwt/jwt"
@@ -15,12 +13,12 @@ import (
 	"github.com/graphql-go/graphql"
 	"github.com/graphql-go/graphql/gqlerrors"
 	"github.com/graphql-go/handler"
-	"github.com/minio/minio-go/v7"
 
 	log "github.com/sirupsen/logrus"
 
 	helper "git.kapelle.org/niklas/s3browser/internal/helper"
 	"git.kapelle.org/niklas/s3browser/internal/loader"
+	"git.kapelle.org/niklas/s3browser/internal/s3"
 	types "git.kapelle.org/niklas/s3browser/internal/types"
 )
 
@@ -112,7 +110,7 @@ func httpGetFile(ctx context.Context, rw http.ResponseWriter, r *http.Request) {
 		return
 	}
 
-	s3Client := ctx.Value("s3Client").(*minio.Client)
+	s3Client := ctx.Value("s3Client").(s3.S3Service)
 	idString := r.URL.Query().Get("id")
 
 	id := types.ParseID(idString)
@@ -123,8 +121,8 @@ func httpGetFile(ctx context.Context, rw http.ResponseWriter, r *http.Request) {
 		return
 	}
 
-	log.Debug("S3 call 'StatObject': ", id)
+	log.Debug("S3 'StatObject': ", id)
-	objInfo, err := s3Client.StatObject(context.Background(), id.Bucket, id.Key, minio.GetObjectOptions{})
+	objInfo, err := s3Client.StatObject(context.Background(), *id)
 
 	if err != nil {
 		log.Error("Failed to get object info: ", err)
@@ -138,8 +136,8 @@ func httpGetFile(ctx context.Context, rw http.ResponseWriter, r *http.Request) {
 		return
 	}
 
-	log.Debug("S3 call 'GetObject': ", id)
+	log.Debug("S3 'GetObject': ", id)
-	obj, err := s3Client.GetObject(context.Background(), id.Bucket, id.Key, minio.GetObjectOptions{})
+	obj, err := s3Client.GetObject(context.Background(), *id)
 
 	if err != nil {
 		log.Error("Failed to get object: ", err)
@@ -148,7 +146,7 @@ func httpGetFile(ctx context.Context, rw http.ResponseWriter, r *http.Request) {
 	}
 
 	rw.Header().Set("Cache-Control", "must-revalidate")
-	rw.Header().Set("Content-Disposition", fmt.Sprintf("attachment; filename=\"%s\"", filepath.Base((objInfo.Key))))
+	rw.Header().Set("Content-Disposition", fmt.Sprintf("attachment; filename=\"%s\"", id.Name()))
 	rw.Header().Set("Content-Type", objInfo.ContentType)
 	rw.Header().Set("ETag", objInfo.ETag)
 
@@ -166,7 +164,7 @@ func httpPostFile(ctx context.Context, rw http.ResponseWriter, r *http.Request)
 		return
 	}
 
-	s3Client := ctx.Value("s3Client").(*minio.Client)
+	s3Client := ctx.Value("s3Client").(s3.S3Service)
 
 	idString := r.URL.Query().Get("id")
 
@@ -180,15 +178,11 @@ func httpPostFile(ctx context.Context, rw http.ResponseWriter, r *http.Request)
 
 	id.Normalize()
 
-	log.Debug("Upload file: ", id)
+	// contentType := r.Header.Get("Content-Type")
+	// mimeType, _, _ := mime.ParseMediaType(contentType)
 
-	contentType := r.Header.Get("Content-Type")
+	log.Debug("S3 'PutObject': ", id)
-	mimeType, _, _ := mime.ParseMediaType(contentType)
+	err := s3Client.PutObject(context.Background(), *id, r.Body, r.ContentLength) // TODO: put content type
-
-	log.Debug("S3 call 'PutObject': ", id)
-	_, err := s3Client.PutObject(context.Background(), id.Bucket, id.Key, r.Body, r.ContentLength, minio.PutObjectOptions{
-		ContentType: mimeType,
-	})
 
 	if err != nil {
 		rw.WriteHeader(http.StatusInternalServerError)
@@ -196,8 +190,7 @@ func httpPostFile(ctx context.Context, rw http.ResponseWriter, r *http.Request)
 	}
 
 	loader := ctx.Value("loader").(*loader.Loader)
-	loader.InvalidateCacheForFile(ctx, *id)
+	loader.InvalidedCacheForId(ctx, *id)
-	loader.InvalidateCacheForDir(ctx, *id.Parent())
 
 	rw.WriteHeader(http.StatusCreated)
 }

internal/httpserver/staticFiles.go

@@ -5,6 +5,7 @@ package httpserver
 
 import (
 	"embed"
+	"github.com/gorilla/mux"
 	"io/fs"
 	"net/http"
 	"os"
@@ -26,7 +27,7 @@ func (spa *spaFileSystem) Open(name string) (http.File, error) {
 	return f, err
 }
 
-func initStatic(e *mux.Router) {
+func initStatic(r *mux.Router) {
 	staticFS, _ := fs.Sub(staticFiles, "static")
-	r.Handle("/", http.FileServer(&spaFileSystem{http.FS(staticFS)}))
+	r.PathPrefix("/").Handler(http.FileServer(&spaFileSystem{http.FS(staticFS)}))
 }

internal/loader/batch.go

@@ -4,18 +4,17 @@ import (
 	"context"
 	"fmt"
 
-	types "git.kapelle.org/niklas/s3browser/internal/types"
+	"git.kapelle.org/niklas/s3browser/internal/s3"
+	"git.kapelle.org/niklas/s3browser/internal/types"
 	"github.com/graph-gophers/dataloader"
-	"github.com/minio/minio-go/v7"
 	log "github.com/sirupsen/logrus"
 )
 
 // listObjectsBatch batch func for calling s3.ListObjects()
 func listObjectsBatch(c context.Context, k dataloader.Keys) []*dataloader.Result {
-	log.Debug("listObjectsBatch: ", k.Keys())
 	var results []*dataloader.Result
 
-	s3Client, ok := c.Value("s3Client").(*minio.Client)
+	s3Client, ok := c.Value("s3Client").(s3.S3Service)
 
 	if !ok {
 		return handleLoaderError(k, fmt.Errorf("Failed to get s3Client from context"))
@@ -23,10 +22,18 @@ func listObjectsBatch(c context.Context, k dataloader.Keys) []*dataloader.Result
 
 	for _, v := range k {
 		id := v.Raw().(types.ID)
-		results = append(results, &dataloader.Result{
+		objects, err := s3Client.ListObjects(c, id)
-			Data:  listObjects(s3Client, id, false),
+		if err != nil {
-			Error: nil,
+			results = append(results, &dataloader.Result{
-		})
+				Data:  nil,
+				Error: err,
+			})
+		} else {
+			results = append(results, &dataloader.Result{
+				Data:  objects,
+				Error: nil,
+			})
+		}
 	}
 
 	return results
@@ -34,10 +41,9 @@ func listObjectsBatch(c context.Context, k dataloader.Keys) []*dataloader.Result
 
 // listObjectsRecursiveBatch just like listObjectsBatch but with recursive set to true
 func listObjectsRecursiveBatch(c context.Context, k dataloader.Keys) []*dataloader.Result {
-	log.Debug("listObjectsRecursiveBatch: ", k.Keys())
 	var results []*dataloader.Result
 
-	s3Client, ok := c.Value("s3Client").(*minio.Client)
+	s3Client, ok := c.Value("s3Client").(s3.S3Service)
 
 	if !ok {
 		return handleLoaderError(k, fmt.Errorf("Failed to get s3Client from context"))
@@ -45,41 +51,33 @@ func listObjectsRecursiveBatch(c context.Context, k dataloader.Keys) []*dataload
 
 	for _, v := range k {
 		id := v.Raw().(types.ID)
-		results = append(results, &dataloader.Result{
+		objects, err := s3Client.ListObjectsRecursive(c, id)
-			Data:  listObjects(s3Client, id, true),
+		if err != nil {
-			Error: nil,
+			results = append(results, &dataloader.Result{
-		})
+				Data:  nil,
+				Error: err,
+			})
+		} else {
+			results = append(results, &dataloader.Result{
+				Data:  objects,
+				Error: nil,
+			})
+		}
 	}
 
 	return results
 }
 
-// listObjects helper func for listObjectsBatch
-func listObjects(s3Client *minio.Client, id types.ID, recursive bool) []minio.ObjectInfo {
-	log.Debug("S3 call 'ListObjects': ", id)
-	objectCh := s3Client.ListObjects(context.Background(), id.Bucket, minio.ListObjectsOptions{
-		Prefix:    id.Key,
-		Recursive: recursive,
-	})
-
-	result := make([]minio.ObjectInfo, 0)
-	for obj := range objectCh {
-		result = append(result, obj)
-	}
-
-	return result
-}
-
 func listBucketsBatch(c context.Context, k dataloader.Keys) []*dataloader.Result {
-	log.Debug("listBucketsBatch")
 	var results []*dataloader.Result
 
-	s3Client, ok := c.Value("s3Client").(*minio.Client)
+	s3Client, ok := c.Value("s3Client").(s3.S3Service)
 
 	if !ok {
 		return handleLoaderError(k, fmt.Errorf("Failed to get s3Client from context"))
 	}
 
+	log.Debug("S3 'ListBuckets'")
 	buckets, err := s3Client.ListBuckets(c)
 
 	if err != nil {
@@ -102,7 +100,7 @@ func statObjectBatch(ctx context.Context, k dataloader.Keys) []*dataloader.Resul
 	log.Debug("statObjectBatch")
 
 	var results []*dataloader.Result
-	s3Client, ok := ctx.Value("s3Client").(*minio.Client)
+	s3Client, ok := ctx.Value("s3Client").(s3.S3Service)
 
 	if !ok {
 		return handleLoaderError(k, fmt.Errorf("Failed to get s3Client from context"))
@@ -110,7 +108,8 @@ func statObjectBatch(ctx context.Context, k dataloader.Keys) []*dataloader.Resul
 
 	for _, v := range k {
 		id := v.Raw().(types.ID)
-		stat, err := s3Client.StatObject(ctx, id.Bucket, id.Key, minio.GetObjectOptions{})
+		log.Debug("S3 'StatObject': ", id)
+		stat, err := s3Client.StatObject(ctx, id)
 		results = append(results, &dataloader.Result{
 			Data:  stat,
 			Error: err,

internal/loader/loader.go

@@ -3,14 +3,12 @@ package loader
 import (
 	"context"
 	"fmt"
-	"path/filepath"
-	"strings"
 
+	"git.kapelle.org/niklas/s3browser/internal/cache"
 	"git.kapelle.org/niklas/s3browser/internal/helper"
+	"git.kapelle.org/niklas/s3browser/internal/s3"
 	types "git.kapelle.org/niklas/s3browser/internal/types"
 	"github.com/graph-gophers/dataloader"
-	"github.com/minio/minio-go/v7"
-	log "github.com/sirupsen/logrus"
 )
 
 type Loader struct {
@@ -18,26 +16,50 @@ type Loader struct {
 	listObjectsRecursiveLoader *dataloader.Loader
 	statObjectLoader           *dataloader.Loader
 	listBucketsLoader          *dataloader.Loader
+
+	listObjectsLoaderCache          cache.S3Cache
+	listObjectsRecursiveLoaderCache cache.S3Cache
+	statObjectLoaderCache           cache.S3Cache
+	listBucketsLoaderCache          cache.S3Cache
 }
 
-func NewLoader(config types.AppConfig) *Loader {
+type CacheConfig struct {
+	ListObjectsLoaderCache          cache.S3Cache
+	ListObjectsRecursiveLoaderCache cache.S3Cache
+	StatObjectLoaderCache           cache.S3Cache
+	ListBucketsLoaderCache          cache.S3Cache
+}
+
+func NewLoader(cacheConfig CacheConfig) *Loader {
+	listObjectsLoaderCache := cacheConfig.ListObjectsLoaderCache
+	listObjectsRecursiveLoaderCache := cacheConfig.ListObjectsRecursiveLoaderCache
+	statObjectLoaderCache := cacheConfig.StatObjectLoaderCache
+	listBucketsLoaderCache := cacheConfig.ListBucketsLoaderCache
+
 	return &Loader{
 		listObjectsLoader: dataloader.NewBatchedLoader(
 			listObjectsBatch,
-			dataloader.WithCache(&dataloader.NoCache{}),
+			dataloader.WithCache(listObjectsLoaderCache),
 		),
+		listObjectsLoaderCache: listObjectsLoaderCache,
+
 		listObjectsRecursiveLoader: dataloader.NewBatchedLoader(
 			listObjectsRecursiveBatch,
-			dataloader.WithCache(&dataloader.NoCache{}),
+			dataloader.WithCache(listObjectsRecursiveLoaderCache),
 		),
+		listObjectsRecursiveLoaderCache: listObjectsRecursiveLoaderCache,
+
 		statObjectLoader: dataloader.NewBatchedLoader(
 			statObjectBatch,
-			dataloader.WithCache(&dataloader.NoCache{}),
+			dataloader.WithCache(statObjectLoaderCache),
 		),
+		statObjectLoaderCache: statObjectLoaderCache,
+
 		listBucketsLoader: dataloader.NewBatchedLoader(
 			listBucketsBatch,
-			dataloader.WithCache(&dataloader.NoCache{}),
+			dataloader.WithCache(listBucketsLoaderCache),
 		),
+		listBucketsLoaderCache: listBucketsLoaderCache,
 	}
 }
 
@@ -51,10 +73,8 @@ func (l *Loader) GetFiles(ctx context.Context, path types.ID) ([]types.File, err
 
 	var files []types.File
 
-	for _, obj := range objects.([]minio.ObjectInfo) {
+	for _, obj := range objects.([]s3.Object) {
-		if obj.Err != nil {
+		if !obj.ID.IsDirectory() {
-			return nil, obj.Err
-		} else if !strings.HasSuffix(obj.Key, "/") {
 			files = append(files, *helper.ObjInfoToFile(obj, path.Bucket))
 		}
 	}
@@ -69,13 +89,13 @@ func (l *Loader) GetFile(ctx context.Context, id types.ID) (*types.File, error)
 	if err != nil {
 		return nil, err
 	}
-	objInfo, ok := result.(minio.ObjectInfo)
+	objInfo, ok := result.(*s3.Object)
 
 	if !ok {
 		return nil, fmt.Errorf("Failed to stats object")
 	}
 
-	return helper.ObjInfoToFile(objInfo, id.Bucket), nil
+	return helper.ObjInfoToFile(*objInfo, id.Bucket), nil
 }
 
 func (l *Loader) GetDirs(ctx context.Context, path types.ID) ([]types.Directory, error) {
@@ -87,20 +107,10 @@ func (l *Loader) GetDirs(ctx context.Context, path types.ID) ([]types.Directory,
 	}
 
 	var dirs []types.Directory
-	for _, obj := range result.([]minio.ObjectInfo) {
+	for _, obj := range result.([]s3.Object) {
-		if obj.Err != nil {
+		if obj.ID.IsDirectory() {
-			return nil, obj.Err
-		} else if strings.HasSuffix(obj.Key, "/") {
-			resultID := types.ID{
-				Bucket: path.Bucket,
-				Key:    obj.Key,
-			}
-
-			resultID.Normalize()
-
 			dirs = append(dirs, types.Directory{
-				ID:   resultID,
+				ID: obj.ID,
-				Name: filepath.Base(obj.Key),
 			})
 		}
 	}
@@ -117,13 +127,7 @@ func (l *Loader) GetBuckets(ctx context.Context) ([]string, error) {
 		return nil, err
 	}
 
-	bucketsInfo := result.([]minio.BucketInfo)
+	return result.([]string), nil
-	var buckets []string
-	for _, i := range bucketsInfo {
-		buckets = append(buckets, i.Name)
-	}
-
-	return buckets, nil
 }
 
 func (l *Loader) GetFilesRecursive(ctx context.Context, path types.ID) ([]types.File, error) {
@@ -134,7 +138,7 @@ func (l *Loader) GetFilesRecursive(ctx context.Context, path types.ID) ([]types.
 		return nil, err
 	}
 
-	objects := result.([]minio.ObjectInfo)
+	objects := result.([]s3.Object)
 
 	var files []types.File
 	for _, obj := range objects {
@@ -144,18 +148,17 @@ func (l *Loader) GetFilesRecursive(ctx context.Context, path types.ID) ([]types.
 	return files, nil
 }
 
-func (l *Loader) InvalidateCacheForFile(ctx context.Context, id types.ID) {
+func (l *Loader) InvalidedCacheForId(ctx context.Context, id types.ID) {
-	log.Debug("Clear cache for file: ", id.String())
 	parent := id.Parent()
 
 	l.statObjectLoader.Clear(ctx, id)
+
+	// Code below is useless for now until we use a propper cache for "listObjectsLoader" and "listObjectsRecursiveLoader"
+	// TODO: implement cache invalidation for "listObjectsLoader" and "listObjectsRecursiveLoader"
 	l.listObjectsLoader.Clear(ctx, id).Clear(ctx, parent)
-}
 
-func (l *Loader) InvalidateCacheForDir(ctx context.Context, path types.ID) {
+	// Remove up from recursive list
-	log.Debug("Clear cache for dir: ", path.String())
+	for rParent := parent; rParent != nil; rParent = rParent.Parent() {
-	parent := helper.GetParentDir(path)
+		l.listObjectsRecursiveLoader.Clear(ctx, rParent)
-
+	}
-	l.listObjectsLoader.Clear(ctx, path).Clear(ctx, parent)
-	l.listObjectsRecursiveLoader.Clear(ctx, path).Clear(ctx, parent)
 }

internal/loader/loader_test.go

@@ -0,0 +1,105 @@
+package loader_test
+
+import (
+	"context"
+	"strings"
+	"testing"
+
+	"git.kapelle.org/niklas/s3browser/internal/loader"
+	"git.kapelle.org/niklas/s3browser/internal/s3"
+	"git.kapelle.org/niklas/s3browser/internal/types"
+	"github.com/graph-gophers/dataloader"
+	"github.com/stretchr/testify/assert"
+)
+
+func setup(t *testing.T) (context.Context, *loader.Loader, *assert.Assertions) {
+	assert := assert.New(t)
+	s3, _ := s3.NewMockS3([]string{"bucket1", "bucket2"})
+	loader := loader.NewLoader(loader.CacheConfig{
+		ListObjectsLoaderCache:          &dataloader.NoCache{},
+		ListObjectsRecursiveLoaderCache: &dataloader.NoCache{},
+		StatObjectLoaderCache:           &dataloader.NoCache{},
+		ListBucketsLoaderCache:          &dataloader.NoCache{},
+	})
+
+	fillS3(s3)
+	ctx := context.WithValue(context.Background(), "s3Client", s3)
+
+	return ctx, loader, assert
+}
+
+func fillS3(s3 s3.S3Service) {
+	ctx := context.Background()
+	length := int64(len("content"))
+
+	for _, v := range []string{
+		"bucket1:/file1", "bucket1:/file2", "bucket1:/dir1/file1",
+		"bucket1:/dir1/file2", "bucket1:/dir2/file1", "bucket1:/dir1/sub1/file1",
+		"bucket1:/dir1/sub1/file2",
+	} {
+		s3.PutObject(ctx, *types.ParseID(v), strings.NewReader("content"), length)
+	}
+}
+
+func TestCreateLoader(t *testing.T) {
+	assert := assert.New(t)
+	loader := loader.NewLoader(loader.CacheConfig{
+		ListObjectsLoaderCache:          &dataloader.NoCache{},
+		ListObjectsRecursiveLoaderCache: &dataloader.NoCache{},
+		StatObjectLoaderCache:           &dataloader.NoCache{},
+		ListBucketsLoaderCache:          &dataloader.NoCache{},
+	})
+
+	assert.NotNil(loader)
+}
+
+func TestGetBuckets(t *testing.T) {
+	ctx, loader, assert := setup(t)
+
+	buckets, err := loader.GetBuckets(ctx)
+	assert.NoError(err)
+	assert.Len(buckets, 2)
+	assert.Contains(buckets, "bucket1")
+	assert.Contains(buckets, "bucket2")
+}
+
+func TestGetFile(t *testing.T) {
+	ctx, loader, assert := setup(t)
+
+	file, err := loader.GetFile(ctx, *types.ParseID("bucket1:/dir1/file1"))
+	assert.NoError(err)
+
+	assert.Equal("bucket1:/dir1/file1", file.ID.String())
+	assert.Equal("file1", file.Name)
+	assert.Equal(int64(len("content")), file.Size)
+}
+
+func TestGetFiles(t *testing.T) {
+	ctx, loader, assert := setup(t)
+
+	id := types.ParseID("bucket1:/")
+
+	files, err := loader.GetFiles(ctx, *id)
+	assert.NoError(err)
+	assert.Len(files, 2)
+}
+
+func TestGetDir(t *testing.T) {
+	ctx, loader, assert := setup(t)
+
+	id := types.ParseID("bucket1:/")
+
+	dirs, err := loader.GetDirs(ctx, *id)
+	assert.NoError(err)
+	assert.Len(dirs, 2)
+}
+
+func Test(t *testing.T) {
+	ctx, loader, assert := setup(t)
+
+	id := types.ParseID("bucket1:/dir1/")
+
+	files, err := loader.GetFilesRecursive(ctx, *id)
+	assert.NoError(err)
+	assert.Len(files, 4)
+}

internal/s3/minio.go

@@ -0,0 +1,131 @@
+package s3
+
+import (
+	"context"
+	"io"
+
+	"git.kapelle.org/niklas/s3browser/internal/types"
+	"github.com/minio/minio-go/v7"
+	"github.com/minio/minio-go/v7/pkg/credentials"
+)
+
+type minioS3 struct {
+	client *minio.Client
+}
+
+func NewMinio(config types.AppConfig) (S3Service, error) {
+	client, err := minio.New(config.S3Endoint, &minio.Options{
+		Creds:  credentials.NewStaticV4(config.S3AccessKey, config.S3SecretKey, ""),
+		Secure: config.S3SSL,
+	})
+
+	if err != nil {
+		return nil, err
+	}
+
+	return &minioS3{
+		client: client,
+	}, nil
+}
+
+func (m *minioS3) ListBuckets(ctx context.Context) ([]string, error) {
+	buckets, err := m.client.ListBuckets(ctx)
+	if err != nil {
+		return nil, err
+	}
+	var rtn []string
+
+	for _, v := range buckets {
+		rtn = append(rtn, v.Name)
+	}
+
+	return rtn, nil
+}
+
+func (m *minioS3) ListObjects(ctx context.Context, id types.ID) ([]Object, error) {
+	var result []Object
+
+	for objInfo := range m.client.ListObjects(ctx, id.Bucket, minio.ListObjectsOptions{
+		Prefix:    id.Key,
+		Recursive: false,
+	}) {
+		objId := types.ID{
+			Bucket: id.Bucket,
+			Key:    objInfo.Key,
+		}
+
+		result = append(result, Object{
+			ID:   objId,
+			Size: objInfo.Size,
+		})
+	}
+
+	return result, nil
+}
+
+func (m *minioS3) ListObjectsRecursive(ctx context.Context, id types.ID) ([]Object, error) {
+	var result []Object
+
+	for objInfo := range m.client.ListObjects(ctx, id.Bucket, minio.ListObjectsOptions{
+		Prefix:    id.Key,
+		Recursive: true,
+	}) {
+		objId := types.ID{
+			Bucket: id.Bucket,
+			Key:    objInfo.Key,
+		}
+		result = append(result, Object{
+			ID:   objId,
+			Size: objInfo.Size,
+		})
+	}
+
+	return result, nil
+}
+
+func (m *minioS3) GetObject(ctx context.Context, id types.ID) (ObjectReader, error) {
+	object, err := m.client.GetObject(ctx, id.Bucket, id.Key, minio.GetObjectOptions{})
+
+	if err != nil {
+		return nil, err
+	}
+
+	return object, nil
+}
+
+func (m *minioS3) PutObject(ctx context.Context, id types.ID, reader io.Reader, objectSize int64) error {
+	_, err := m.client.PutObject(ctx, id.Bucket, id.Key, reader, objectSize, minio.PutObjectOptions{})
+	return err
+}
+
+func (m *minioS3) CopyObject(ctx context.Context, src types.ID, dest types.ID) error {
+	_, err := m.client.CopyObject(ctx, minio.CopyDestOptions{
+		Bucket: dest.Bucket,
+		Object: dest.Key,
+	}, minio.CopySrcOptions{
+		Bucket: src.Bucket,
+		Object: src.Key,
+	})
+
+	return err
+}
+
+func (m *minioS3) StatObject(ctx context.Context, id types.ID) (*Object, error) {
+	info, err := m.client.StatObject(ctx, id.Bucket, id.Key, minio.GetObjectOptions{})
+
+	if err != nil {
+		return nil, err
+	}
+
+	return &Object{
+		ID:           id,
+		Size:         info.Size,
+		LastModified: info.LastModified,
+		ContentType:  info.ContentType,
+		ETag:         info.ETag,
+	}, nil
+}
+
+func (m *minioS3) RemoveObject(ctx context.Context, id types.ID) error {
+	return m.client.RemoveObject(ctx, id.Bucket, id.Key, minio.RemoveObjectOptions{})
+}

internal/s3/mock.go

@@ -0,0 +1,154 @@
+package s3
+
+import (
+	"bytes"
+	"context"
+	"crypto/md5"
+	"fmt"
+	"io"
+	"io/ioutil"
+	"strings"
+	"time"
+
+	"git.kapelle.org/niklas/s3browser/internal/types"
+)
+
+type mockS3 struct {
+	buckets []string
+	objects map[types.ID]mockObject
+}
+
+type mockObject struct {
+	content     []byte
+	contentType string
+	lastMod     time.Time
+}
+
+type mockObjectReader struct {
+	*bytes.Reader
+}
+
+func (r mockObjectReader) Close() error {
+	// NOOP
+	return nil
+}
+
+func NewMockS3(buckets []string) (S3Service, error) {
+	return &mockS3{
+		buckets: buckets,
+		objects: map[types.ID]mockObject{},
+	}, nil
+}
+
+func (m *mockS3) ListBuckets(ctx context.Context) ([]string, error) {
+	return m.buckets, nil
+}
+
+func (m *mockS3) ListObjects(ctx context.Context, id types.ID) ([]Object, error) {
+	var results []Object
+
+	dirs := make(map[string]bool)
+
+	depth := len(strings.Split(id.Key, "/"))
+
+	for k, v := range m.objects {
+		if k.Bucket == id.Bucket {
+			if k.Parent().Key == id.Key {
+				results = append(results, *mockObjToObject(v, k))
+			} else if strings.HasPrefix(k.Key, id.Key) {
+				s := strings.Join(strings.Split(k.Key, "/")[:depth], "/") + "/"
+				dirs[s] = true
+			}
+		}
+	}
+
+	for k := range dirs {
+		results = append(results, Object{
+			ID: types.ID{
+				Bucket: id.Bucket,
+				Key:    k,
+			},
+		})
+	}
+
+	return results, nil
+}
+
+func (m *mockS3) ListObjectsRecursive(ctx context.Context, id types.ID) ([]Object, error) {
+	var results []Object
+
+	for k, v := range m.objects {
+		if k.Bucket == id.Bucket {
+			if strings.HasPrefix(k.Key, id.Key) {
+				results = append(results, *mockObjToObject(v, k))
+			}
+		}
+	}
+
+	return results, nil
+}
+
+func (m *mockS3) GetObject(ctx context.Context, id types.ID) (ObjectReader, error) {
+	mockObj, exist := m.objects[id]
+
+	if !exist {
+		return nil, fmt.Errorf("Object not found")
+	}
+
+	reader := bytes.NewReader(mockObj.content)
+
+	return mockObjectReader{reader}, nil
+}
+
+func (m *mockS3) PutObject(ctx context.Context, id types.ID, reader io.Reader, objectSize int64) error {
+	content, err := ioutil.ReadAll(reader)
+
+	if err != nil {
+		return err
+	}
+
+	m.objects[id] = mockObject{
+		content:     content,
+		lastMod:     time.Now(),
+		contentType: "application/octet-stream", // TODO: detect MIME type or dont its just a mock after all
+	}
+
+	return nil
+}
+
+func (m *mockS3) CopyObject(ctx context.Context, src types.ID, dest types.ID) error {
+	srcObj, exist := m.objects[src]
+
+	if !exist {
+		return fmt.Errorf("Object not found")
+	}
+
+	m.objects[dest] = srcObj
+
+	return nil
+}
+
+func (m *mockS3) StatObject(ctx context.Context, id types.ID) (*Object, error) {
+	mockObj, exist := m.objects[id]
+
+	if !exist {
+		return nil, fmt.Errorf("Object not found")
+	}
+
+	return mockObjToObject(mockObj, id), nil
+}
+
+func (m *mockS3) RemoveObject(ctx context.Context, id types.ID) error {
+	delete(m.objects, id)
+	return nil
+}
+
+func mockObjToObject(mockObj mockObject, id types.ID) *Object {
+	return &Object{
+		ID:           id,
+		Size:         int64(len(mockObj.content)),
+		ContentType:  mockObj.contentType,
+		LastModified: mockObj.lastMod,
+		ETag:         fmt.Sprintf("%x", md5.Sum(mockObj.content)),
+	}
+}

internal/s3/mock_test.go

@@ -0,0 +1,162 @@
+package s3_test
+
+import (
+	"context"
+	"io/ioutil"
+	"strings"
+	"testing"
+	"time"
+
+	"git.kapelle.org/niklas/s3browser/internal/s3"
+	"git.kapelle.org/niklas/s3browser/internal/types"
+	"github.com/stretchr/testify/assert"
+)
+
+func setup(t *testing.T) (s3.S3Service, context.Context, *assert.Assertions) {
+	service, _ := s3.NewMockS3([]string{"bucket1", "bucket2"})
+	ctx := context.Background()
+	assert := assert.New(t)
+
+	return service, ctx, assert
+}
+
+func TestBuckets(t *testing.T) {
+	s3, ctx, assert := setup(t)
+
+	buckets, err := s3.ListBuckets(ctx)
+	assert.NoError(err)
+
+	assert.Len(buckets, 2)
+
+	assert.Contains(buckets, "bucket1")
+	assert.Contains(buckets, "bucket2")
+}
+
+func TestPut(t *testing.T) {
+	s3, ctx, assert := setup(t)
+	content := "FileContent"
+
+	err := s3.PutObject(ctx, *types.ParseID("bucket1:/file1"), strings.NewReader(content), int64(len(content)))
+	assert.NoError(err)
+}
+
+func TestPutAndGet(t *testing.T) {
+	s3, ctx, assert := setup(t)
+
+	content := "FileContent"
+	id := *types.ParseID("bucket1:/file1")
+
+	err := s3.PutObject(ctx, *types.ParseID("bucket1:/file1"), strings.NewReader(content), int64(len(content)))
+	assert.NoError(err)
+
+	reader, err := s3.GetObject(ctx, id)
+	assert.NoError(err)
+
+	readerContent, err := ioutil.ReadAll(reader)
+	assert.NoError(err)
+
+	assert.Equal(content, string(readerContent))
+}
+
+func TestStat(t *testing.T) {
+	s3, ctx, assert := setup(t)
+
+	content := "FileContent"
+	id := *types.ParseID("bucket1:/file1")
+
+	now := time.Now()
+
+	err := s3.PutObject(ctx, id, strings.NewReader(content), int64(len(content)))
+	assert.NoError(err)
+
+	obj, err := s3.StatObject(ctx, id)
+	assert.NoError(err)
+
+	assert.Equal(id.String(), obj.ID.String())
+	assert.Equal(int64(len(content)), obj.Size)
+	assert.NotEmpty(obj.ETag)
+	assert.WithinDuration(now, obj.LastModified, time.Second*1)
+}
+
+func TestRemove(t *testing.T) {
+	s3, ctx, assert := setup(t)
+
+	content := "FileContent"
+	id := *types.ParseID("bucket1:/file1")
+
+	err := s3.PutObject(ctx, id, strings.NewReader(content), int64(len(content)))
+	assert.NoError(err)
+
+	err = s3.RemoveObject(ctx, id)
+	assert.NoError(err)
+
+	_, err = s3.StatObject(ctx, id)
+	assert.Error(err)
+}
+
+func TestList(t *testing.T) {
+	s3, ctx, assert := setup(t)
+
+	content1 := "FileContent1"
+	id1 := *types.ParseID("bucket1:/file1")
+
+	err := s3.PutObject(ctx, id1, strings.NewReader(content1), int64(len(content1)))
+	assert.NoError(err)
+
+	content2 := "FileContent2"
+	id2 := *types.ParseID("bucket1:/file2")
+
+	err = s3.PutObject(ctx, id2, strings.NewReader(content2), int64(len(content2)))
+	assert.NoError(err)
+
+	listID := types.ParseID("bucket1:/")
+
+	objects, err := s3.ListObjects(ctx, *listID)
+	assert.NoError(err)
+
+	assert.Len(objects, 2)
+}
+
+func TestListRecursive(t *testing.T) {
+	s3, ctx, assert := setup(t)
+
+	s3.PutObject(ctx, *types.ParseID("bucket1:/file1"), strings.NewReader("content"), int64(len("content")))
+	s3.PutObject(ctx, *types.ParseID("bucket1:/path1/file1"), strings.NewReader("content"), int64(len("content")))
+	s3.PutObject(ctx, *types.ParseID("bucket1:/path1/file2"), strings.NewReader("content"), int64(len("content")))
+	s3.PutObject(ctx, *types.ParseID("bucket1:/path1/path2/file1"), strings.NewReader("content"), int64(len("content")))
+	s3.PutObject(ctx, *types.ParseID("bucket1:/path3/path4/file1"), strings.NewReader("content"), int64(len("content")))
+
+	objects, err := s3.ListObjectsRecursive(ctx, *types.ParseID("bucket1:/path1/"))
+	assert.NoError(err)
+	assert.Len(objects, 3)
+}
+
+func TestCopy(t *testing.T) {
+	s3, ctx, assert := setup(t)
+
+	id1 := *types.ParseID("bucket1:/file1")
+	id2 := *types.ParseID("bucket1:/file2")
+
+	s3.PutObject(ctx, id1, strings.NewReader("content"), int64(len("content")))
+
+	err := s3.CopyObject(ctx, id1, id2)
+	assert.NoError(err)
+
+	obj1, err := s3.StatObject(ctx, id1)
+	assert.NoError(err)
+	assert.NotNil(obj1)
+
+	obj2, err := s3.StatObject(ctx, id1)
+	assert.NoError(err)
+	assert.NotNil(obj2)
+
+	assert.Equal(obj1.ETag, obj2.ETag)
+	assert.Equal(obj1.Size, obj2.Size)
+
+	obj2Reader, err := s3.GetObject(ctx, id2)
+	assert.NoError(err)
+
+	obj2Content, err := ioutil.ReadAll(obj2Reader)
+	assert.NoError(err)
+	assert.Equal([]byte("content"), obj2Content)
+}

internal/s3/s3.go

@@ -0,0 +1,37 @@
+package s3
+
+import (
+	"context"
+	"io"
+	"time"
+
+	"git.kapelle.org/niklas/s3browser/internal/types"
+)
+
+type ObjectReader interface {
+	io.Reader
+	io.Seeker
+	io.ReaderAt
+	io.Closer
+}
+
+type Object struct {
+	ID           types.ID
+	Size         int64
+	LastModified time.Time
+	ContentType  string
+	ETag         string
+}
+
+type S3Service interface {
+	ListBuckets(ctx context.Context) ([]string, error)
+
+	GetObject(ctx context.Context, id types.ID) (ObjectReader, error)
+	PutObject(ctx context.Context, id types.ID, reader io.Reader, objectSize int64) error
+
+	ListObjects(ctx context.Context, id types.ID) ([]Object, error)
+	ListObjectsRecursive(ctx context.Context, id types.ID) ([]Object, error)
+	CopyObject(ctx context.Context, src types.ID, dest types.ID) error
+	StatObject(ctx context.Context, id types.ID) (*Object, error)
+	RemoveObject(ctx context.Context, id types.ID) error
+}

internal/s3Broswer.go

@@ -2,43 +2,19 @@ package s3browser
 
 import (
 	"context"
-	"fmt"
 
-	"github.com/minio/minio-go/v7"
+	"github.com/graph-gophers/dataloader"
-	"github.com/minio/minio-go/v7/pkg/credentials"
 	log "github.com/sirupsen/logrus"
 
+	"git.kapelle.org/niklas/s3browser/internal/cache"
 	"git.kapelle.org/niklas/s3browser/internal/db"
 	gql "git.kapelle.org/niklas/s3browser/internal/gql"
 	httpserver "git.kapelle.org/niklas/s3browser/internal/httpserver"
 	"git.kapelle.org/niklas/s3browser/internal/loader"
+	"git.kapelle.org/niklas/s3browser/internal/s3"
 	types "git.kapelle.org/niklas/s3browser/internal/types"
 )
 
-// setupS3Client connect the s3Client
-func setupS3Client(config types.AppConfig) (*minio.Client, error) {
-	minioClient, err := minio.New(config.S3Endoint, &minio.Options{
-		Creds:  credentials.NewStaticV4(config.S3AccessKey, config.S3SecretKey, ""),
-		Secure: config.S3SSL,
-	})
-
-	if err != nil {
-		return nil, err
-	}
-
-	exists, err := minioClient.BucketExists(context.Background(), config.S3Bucket)
-
-	if err != nil {
-		return nil, err
-	}
-
-	if !exists {
-		return nil, fmt.Errorf("Bucket '%s' does not exist", config.S3Bucket)
-	}
-
-	return minioClient, nil
-}
-
 // Start starts the app
 func Start(config types.AppConfig) {
 
@@ -47,24 +23,25 @@ func Start(config types.AppConfig) {
 	}
 
 	log.Info("Starting")
-	log.Debug("Setting up s3 client")
+	s3Client, err := s3.NewMinio(config)
-	s3Client, err := setupS3Client(config)
 
 	if err != nil {
 		log.Error("Failed to setup s3 client: ", err.Error())
 		return
 	}
-	log.Info("s3 client connected")
 
-	dbStore, err := db.NewDB("mysql", "s3Browser:hunter2@/s3Browser")
+	dbStore, err := db.NewDB(config.DSN)
 	if err != nil {
 		log.Error("Failed to connect DB: ", err.Error())
 	}
 
-	log.Debug("Creating dataloader")
+	loader := loader.NewLoader(loader.CacheConfig{
-	loader := loader.NewLoader(config)
+		ListObjectsLoaderCache:          &dataloader.NoCache{},
+		ListObjectsRecursiveLoaderCache: &dataloader.NoCache{},
+		StatObjectLoaderCache:           cache.NewTTLCache(config.CacheTTL, config.CacheCleanup),
+		ListBucketsLoaderCache:          cache.NewTTLCache(config.CacheTTL, config.CacheCleanup),
+	})
 
-	log.Debug("Generating graphq schema")
 	gql.GraphqlTypes()
 	schema, err := gql.GraphqlSchema()
 
@@ -77,7 +54,6 @@ func Start(config types.AppConfig) {
 	resolveContext = context.WithValue(resolveContext, "loader", loader)
 	resolveContext = context.WithValue(resolveContext, "dbStore", dbStore)
 
-	log.Debug("Starting HTTP server")
 	err = httpserver.InitHttp(resolveContext, schema, config.Address)
 
 	if err != nil {

internal/types/id.go

@@ -69,9 +69,15 @@ func (i ID) Parent() *ID {
 			Key:    strings.Join(parts[:len(parts)-2], "/") + "/",
 		}
 	} else {
+		dir := filepath.Dir(i.Key)
+
+		if dir != "/" {
+			dir += "/"
+		}
+
 		parent = &ID{
 			Bucket: i.Bucket,
-			Key:    filepath.Dir(i.Key) + "/",
+			Key:    dir,
 		}
 	}
 
@@ -80,6 +86,11 @@ func (i ID) Parent() *ID {
 	return parent
 }
 
+// Name returns filename or directory name
+func (i ID) Name() string {
+	return filepath.Base(i.Key)
+}
+
 // ParseID parses a string to an ID. Null if invalid
 func ParseID(id string) *ID {
 	match := idRegex.FindStringSubmatch(id)

internal/types/id_test.go

@@ -0,0 +1,109 @@
+package types_test
+
+import (
+	"testing"
+
+	"git.kapelle.org/niklas/s3browser/internal/types"
+	"github.com/stretchr/testify/assert"
+)
+
+// TODO: test version component (not yet used in code)
+
+func TestIDParse(t *testing.T) {
+	assert := assert.New(t)
+
+	id := types.ParseID("test:/path/key")
+
+	assert.NotNil(id)
+	assert.True(id.Valid())
+	assert.Equal("test", id.Bucket)
+	assert.Equal("/path/key", id.Key)
+	assert.False(id.IsDirectory())
+	assert.Equal("key", id.Name())
+	assert.Equal("test:/path/key", id.String())
+}
+
+func TestIDParseInvalid(t *testing.T) {
+	assert := assert.New(t)
+
+	assert.Nil(types.ParseID("/asd/ad"))
+	assert.Nil(types.ParseID("test"))
+	assert.Nil(types.ParseID("test:"))
+	assert.Nil(types.ParseID(""))
+	assert.Nil(types.ParseID("/"))
+}
+
+func TestIDIsDir(t *testing.T) {
+	assert := assert.New(t)
+
+	idFile := types.ParseID("test:/path/key")
+	assert.NotNil(idFile)
+	assert.False(idFile.IsDirectory())
+
+	idDir := types.ParseID("test:/path/key/")
+	assert.NotNil(idDir)
+	assert.True(idDir.IsDirectory())
+}
+
+func TestIDRoot(t *testing.T) {
+	assert := assert.New(t)
+
+	id := types.ParseID("test:/")
+
+	assert.NotNil(id)
+	assert.True(id.Valid())
+	assert.Equal("test", id.Bucket)
+	assert.Equal("/", id.Key)
+	assert.True(id.IsDirectory())
+	assert.Equal("/", id.Name())
+	assert.Equal("test:/", id.String())
+
+	assert.Nil(id.Parent())
+}
+
+func TestIDParentFromFile(t *testing.T) {
+	assert := assert.New(t)
+
+	id := types.ParseID("test:/path1/path2/key")
+
+	assert.NotNil(id)
+
+	parent := id.Parent()
+
+	assert.NotNil(parent)
+	assert.True(parent.Valid())
+	assert.Equal("test", parent.Bucket)
+	assert.Equal("/path1/path2/", parent.Key)
+	assert.True(parent.IsDirectory())
+	assert.Equal("path2", parent.Name())
+	assert.Equal("test:/path1/path2/", parent.String())
+}
+
+func TestIDParentFromDir(t *testing.T) {
+	assert := assert.New(t)
+
+	id := types.ParseID("test:/path1/path2/")
+
+	assert.NotNil(id)
+
+	parent := id.Parent()
+
+	assert.NotNil(parent)
+	assert.True(parent.Valid())
+	assert.Equal("test", parent.Bucket)
+	assert.Equal("/path1/", parent.Key)
+	assert.True(parent.IsDirectory())
+	assert.Equal("path1", parent.Name())
+	assert.Equal("test:/path1/", parent.String())
+}
+
+func TestIDParentRoot(t *testing.T) {
+	assert := assert.New(t)
+
+	id := types.ParseID("test:/key1")
+
+	parent := id.Parent()
+
+	assert.NotNil(parent)
+	assert.Equal("/", parent.Key)
+}

internal/types/types.go

@@ -12,7 +12,7 @@ type AppConfig struct {
 	S3AccessKey  string
 	S3SecretKey  string
 	S3SSL        bool
-	S3Bucket     string
+	DSN          string
 	CacheTTL     time.Duration
 	CacheCleanup time.Duration
 	Address      string