From d88552c57f08d7186b8b9248fb756e7fec6c6a40 Mon Sep 17 00:00:00 2001 From: Djeeberjr Date: Mon, 9 May 2022 14:52:18 +0200 Subject: [PATCH] inital commit --- cmd/s3share.go | 7 ++++ go.mod | 29 +++++++++++++ go.sum | 59 +++++++++++++++++++++++++++ internal/client/client.go | 81 +++++++++++++++++++++++++++++++++++++ internal/db/db.go | 14 +++++++ internal/db/sqlite.go | 83 ++++++++++++++++++++++++++++++++++++++ internal/db/sqlite.sql | 5 +++ internal/s3/minio.go | 41 +++++++++++++++++++ internal/s3/s3.go | 17 ++++++++ internal/s3Share.go | 34 ++++++++++++++++ internal/types/share.go | 6 +++ internal/web/web.go | 85 +++++++++++++++++++++++++++++++++++++++ 12 files changed, 461 insertions(+) create mode 100644 cmd/s3share.go create mode 100644 go.mod create mode 100644 go.sum create mode 100644 internal/client/client.go create mode 100644 internal/db/db.go create mode 100644 internal/db/sqlite.go create mode 100644 internal/db/sqlite.sql create mode 100644 internal/s3/minio.go create mode 100644 internal/s3/s3.go create mode 100644 internal/s3Share.go create mode 100644 internal/types/share.go create mode 100644 internal/web/web.go diff --git a/cmd/s3share.go b/cmd/s3share.go new file mode 100644 index 0000000..1477582 --- /dev/null +++ b/cmd/s3share.go @@ -0,0 +1,7 @@ +package main + +import s3share "git.kapelle.org/niklas/s3share/internal" + +func main() { + s3share.Start() +} diff --git a/go.mod b/go.mod new file mode 100644 index 0000000..7a284e2 --- /dev/null +++ b/go.mod @@ -0,0 +1,29 @@ +module git.kapelle.org/niklas/s3share + +go 1.18 + +require ( + github.com/gorilla/mux v1.8.0 + github.com/mattn/go-sqlite3 v1.14.12 +) + +require ( + github.com/dustin/go-humanize v1.0.0 // indirect + github.com/google/uuid v1.1.1 // indirect + github.com/json-iterator/go v1.1.12 // indirect + github.com/klauspost/compress v1.13.5 // indirect + github.com/klauspost/cpuid v1.3.1 // indirect + github.com/minio/md5-simd v1.1.0 // indirect + github.com/minio/minio-go/v7 v7.0.26 // indirect + github.com/minio/sha256-simd v0.1.1 // indirect + github.com/mitchellh/go-homedir v1.1.0 // indirect + github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect + github.com/modern-go/reflect2 v1.0.2 // indirect + github.com/rs/xid v1.2.1 // indirect + github.com/sirupsen/logrus v1.8.1 // indirect + golang.org/x/crypto v0.0.0-20201216223049-8b5274cf687f // indirect + golang.org/x/net v0.0.0-20200707034311-ab3426394381 // indirect + golang.org/x/sys v0.0.0-20200625212154-ddb9806d33ae // indirect + golang.org/x/text v0.3.3 // indirect + gopkg.in/ini.v1 v1.57.0 // indirect +) diff --git a/go.sum b/go.sum new file mode 100644 index 0000000..4d8d544 --- /dev/null +++ b/go.sum @@ -0,0 +1,59 @@ +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/dustin/go-humanize v1.0.0 h1:VSnTsYCnlFHaM2/igO1h6X3HA71jcobQuxemgkq4zYo= +github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= +github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= +github.com/google/uuid v1.1.1 h1:Gkbcsh/GbpXz7lPftLA3P6TYMwjCLYm83jiFQZF/3gY= +github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/gorilla/mux v1.8.0 h1:i40aqfkR1h2SlN9hojwV5ZA91wcXFOvkdNIeFDP5koI= +github.com/gorilla/mux v1.8.0/go.mod h1:DVbg23sWSpFRCP0SfiEN6jmj59UnW/n46BH5rLB71So= +github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= +github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= +github.com/klauspost/compress v1.13.5 h1:9O69jUPDcsT9fEm74W92rZL9FQY7rCdaXVneq+yyzl4= +github.com/klauspost/compress v1.13.5/go.mod h1:/3/Vjq9QcHkK5uEr5lBEmyoZ1iFhe47etQ6QUkpK6sk= +github.com/klauspost/cpuid v1.2.3/go.mod h1:Pj4uuM528wm8OyEC2QMXAi2YiTZ96dNQPGgoMS4s3ek= +github.com/klauspost/cpuid v1.3.1 h1:5JNjFYYQrZeKRJ0734q51WCEEn2huer72Dc7K+R/b6s= +github.com/klauspost/cpuid v1.3.1/go.mod h1:bYW4mA6ZgKPob1/Dlai2LviZJO7KGI3uoWLd42rAQw4= +github.com/mattn/go-sqlite3 v1.14.12 h1:TJ1bhYJPV44phC+IMu1u2K/i5RriLTPe+yc68XDJ1Z0= +github.com/mattn/go-sqlite3 v1.14.12/go.mod h1:NyWgC/yNuGj7Q9rpYnZvas74GogHl5/Z4A/KQRfk6bU= +github.com/minio/md5-simd v1.1.0 h1:QPfiOqlZH+Cj9teu0t9b1nTBfPbyTl16Of5MeuShdK4= +github.com/minio/md5-simd v1.1.0/go.mod h1:XpBqgZULrMYD3R+M28PcmP0CkI7PEMzB3U77ZrKZ0Gw= +github.com/minio/minio-go/v7 v7.0.26 h1:D0HK+8793etZfRY/vHhDmFaP+vmT41K3K4JV9vmZCBQ= +github.com/minio/minio-go/v7 v7.0.26/go.mod h1:x81+AX5gHSfCSqw7jxRKHvxUXMlE5uKX0Vb75Xk5yYg= +github.com/minio/sha256-simd v0.1.1 h1:5QHSlgo3nt5yKOJrC7W8w7X+NFl8cMPZm96iu8kKUJU= +github.com/minio/sha256-simd v0.1.1/go.mod h1:B5e1o+1/KgNmWrSQK08Y6Z1Vb5pwIktudl0J58iy0KM= +github.com/mitchellh/go-homedir v1.1.0 h1:lukF9ziXFxDFPkA1vsr5zpc1XuPDn/wFntq5mG+4E0Y= +github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= +github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= +github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= +github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= +github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M= +github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/rs/xid v1.2.1 h1:mhH9Nq+C1fY2l1XIpgxIiUOfNpRBYH1kKcr+qfKgjRc= +github.com/rs/xid v1.2.1/go.mod h1:+uKXf+4Djp6Md1KODXJxgGQPKngRmWyn10oCKFzNHOQ= +github.com/sirupsen/logrus v1.8.1 h1:dJKuHgqk1NNQlqoA6BTlM1Wf9DOH3NBjQyu0h9+AZZE= +github.com/sirupsen/logrus v1.8.1/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= +github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= +golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/crypto v0.0.0-20201216223049-8b5274cf687f h1:aZp0e2vLN4MToVqnjNEYEtrEA8RH8U8FN1CU7JgqsPU= +golang.org/x/crypto v0.0.0-20201216223049-8b5274cf687f/go.mod h1:jdWPYTVW3xRLrWPugEBEK3UY2ZEsg3UU495nc5E+M+I= +golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20200707034311-ab3426394381 h1:VXak5I6aEWmAXeQjA+QSZzlgNrpq9mjcfDemuexIKsU= +golang.org/x/net v0.0.0-20200707034311-ab3426394381/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= +golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200625212154-ddb9806d33ae h1:Ih9Yo4hSPImZOpfGuA4bR/ORKTAbhZo2AbWNRCnevdo= +golang.org/x/sys v0.0.0-20200625212154-ddb9806d33ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw= +golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.3 h1:cokOdA+Jmi5PJGXLlLllQSgYigAEfHXJAERHVMaCc2k= +golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +gopkg.in/ini.v1 v1.57.0 h1:9unxIsFcTt4I55uWluz+UmL95q4kdJ0buvQ1ZIqVQww= +gopkg.in/ini.v1 v1.57.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= diff --git a/internal/client/client.go b/internal/client/client.go new file mode 100644 index 0000000..62954b5 --- /dev/null +++ b/internal/client/client.go @@ -0,0 +1,81 @@ +package client + +import ( + "context" + "errors" + "math/rand" + "time" + + "git.kapelle.org/niklas/s3share/internal/db" + "git.kapelle.org/niklas/s3share/internal/s3" + "git.kapelle.org/niklas/s3share/internal/types" +) + +var letters = []rune("abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789") + +type Client struct { + db db.DB + s3 s3.S3 +} + +func NewClient(db db.DB, s3 s3.S3) *Client { + rand.Seed(time.Now().UnixNano()) + return &Client{ + db: db, + s3: s3, + } +} + +func createRandomString() string { + s := make([]rune, 6) + for i := range s { + s[i] = letters[rand.Intn(len(letters))] + } + return string(s) +} + +func (c *Client) CreateValidSlug(ctx context.Context) (string, error) { + for i := 0; i < 10; i++ { + slug := createRandomString() + + res, err := c.db.GetShare(ctx, slug) + if err != nil { + return "", err + } + + if res == nil { + return slug, nil + } + } + + return "", errors.New("could not create valid slug after 10 tries") +} + +func (c *Client) GetShare(ctx context.Context, slug string) (*types.Share, error) { + return c.db.GetShare(ctx, slug) +} + +func (c *Client) CreateShare(ctx context.Context, key string) (*types.Share, error) { + slug, err := c.CreateValidSlug(ctx) + if err != nil { + return nil, err + } + + share := &types.Share{ + Slug: slug, + Key: key, + } + + // TODO: check if key exists + + err = c.db.CreateShare(ctx, share) + if err != nil { + return nil, err + } + + return share, nil +} + +func (c *Client) GetObjectFromShare(ctx context.Context, share *types.Share) (s3.ObjectReader, error) { + return c.s3.GetObject(ctx, share.Key) +} diff --git a/internal/db/db.go b/internal/db/db.go new file mode 100644 index 0000000..3d1c3db --- /dev/null +++ b/internal/db/db.go @@ -0,0 +1,14 @@ +package db + +import ( + "context" + + "git.kapelle.org/niklas/s3share/internal/types" +) + +type DB interface { + GetShare(ctx context.Context, slug string) (*types.Share, error) + CreateShare(ctx context.Context, share *types.Share) error + DeleteShare(ctx context.Context, slug string) error + Close() error +} diff --git a/internal/db/sqlite.go b/internal/db/sqlite.go new file mode 100644 index 0000000..62f92b4 --- /dev/null +++ b/internal/db/sqlite.go @@ -0,0 +1,83 @@ +package db + +import ( + "context" + "database/sql" + "os" + + _ "embed" + + _ "github.com/mattn/go-sqlite3" + "github.com/sirupsen/logrus" + + "git.kapelle.org/niklas/s3share/internal/types" +) + +//go:embed sqlite.sql +var setupSql string + +type sqlLiteDB struct { + db *sql.DB +} + +func NewSqlLiteDB(path string) (DB, error) { + os.Remove(path) + + logrus.Info("Opening database") + db, err := sql.Open("sqlite3", path) + if err != nil { + return nil, err + } + + logrus.Info("Initializing database") + _, err = db.Exec(setupSql) + if err != nil { + return nil, err + } + + return &sqlLiteDB{ + db: db, + }, nil +} + +func (db *sqlLiteDB) GetShare(ctx context.Context, slug string) (*types.Share, error) { + res, err := db.db.QueryContext(ctx, "SELECT slug, objKey FROM shares WHERE slug = ?", slug) + if err != nil { + return nil, err + } + + if !res.Next() { + return nil, nil + } + + var share types.Share + + err = res.Scan(&share.Slug, &share.Key) + + if err != nil { + return nil, err + } + + return &share, nil +} + +func (db *sqlLiteDB) CreateShare(ctx context.Context, share *types.Share) error { + _, err := db.db.ExecContext(ctx, "INSERT INTO shares (slug, objKey) VALUES (?, ?)", share.Slug, share.Key) + if err != nil { + return err + } + + return nil +} + +func (db *sqlLiteDB) DeleteShare(ctx context.Context, slug string) error { + _, err := db.db.ExecContext(ctx, "DELETE FROM shares WHERE slug = ?", slug) + if err != nil { + return err + } + return nil +} + +func (db *sqlLiteDB) Close() error { + return db.db.Close() +} diff --git a/internal/db/sqlite.sql b/internal/db/sqlite.sql new file mode 100644 index 0000000..6defb6a --- /dev/null +++ b/internal/db/sqlite.sql @@ -0,0 +1,5 @@ +CREATE TABLE `shares` ( + slug VARCHAR(6) NOT NULL PRIMARY KEY, + objKey TEXT NOT NULL +); + diff --git a/internal/s3/minio.go b/internal/s3/minio.go new file mode 100644 index 0000000..0c80718 --- /dev/null +++ b/internal/s3/minio.go @@ -0,0 +1,41 @@ +package s3 + +import ( + "context" + + "github.com/minio/minio-go/v7" + "github.com/minio/minio-go/v7/pkg/credentials" + "github.com/sirupsen/logrus" +) + +type minioClient struct { + client *minio.Client + bucket string +} + +func NewMinio(endpoint, bucket, accessKey, secretAccessKey string, ssl bool) (S3, error) { + logrus.Info("Creating minio client") + client, err := minio.New(endpoint, &minio.Options{ + Creds: credentials.NewStaticV4(accessKey, secretAccessKey, ""), + Secure: ssl, + }) + + if err != nil { + return nil, err + } + + return &minioClient{ + client: client, + bucket: bucket, + }, nil +} + +func (m *minioClient) GetObject(ctx context.Context, key string) (ObjectReader, error) { + object, err := m.client.GetObject(ctx, m.bucket, key, minio.GetObjectOptions{}) + + if err != nil { + return nil, err + } + + return object, nil +} diff --git a/internal/s3/s3.go b/internal/s3/s3.go new file mode 100644 index 0000000..8da9a9f --- /dev/null +++ b/internal/s3/s3.go @@ -0,0 +1,17 @@ +package s3 + +import ( + "context" + "io" +) + +type ObjectReader interface { + io.Reader + io.Seeker + io.ReaderAt + io.Closer +} + +type S3 interface { + GetObject(ctx context.Context, key string) (ObjectReader, error) +} diff --git a/internal/s3Share.go b/internal/s3Share.go new file mode 100644 index 0000000..00997d7 --- /dev/null +++ b/internal/s3Share.go @@ -0,0 +1,34 @@ +package s3share + +import ( + "git.kapelle.org/niklas/s3share/internal/client" + "git.kapelle.org/niklas/s3share/internal/db" + "git.kapelle.org/niklas/s3share/internal/s3" + "git.kapelle.org/niklas/s3share/internal/web" +) + +func Start() { + db, err := db.NewSqlLiteDB("foo.db") + if err != nil { + panic(err) + } + + s3Client, err := s3.NewMinio("localhost:9000", "testo", "testo", "hunter22", false) + + if err != nil { + panic(err) + } + + client := client.NewClient(db, s3Client) + + // share, err := client.CreateShare(context.Background(), "/go.mod") + // if err != nil { + // panic(err) + // } + // logrus.Info(share.Slug) + + err = web.StartWebserver("localhost:8080", *client) + if err != nil { + panic(err) + } +} diff --git a/internal/types/share.go b/internal/types/share.go new file mode 100644 index 0000000..2f8f8f9 --- /dev/null +++ b/internal/types/share.go @@ -0,0 +1,6 @@ +package types + +type Share struct { + Slug string `json:"slug"` + Key string `json:"key"` +} diff --git a/internal/web/web.go b/internal/web/web.go new file mode 100644 index 0000000..f8ad486 --- /dev/null +++ b/internal/web/web.go @@ -0,0 +1,85 @@ +package web + +import ( + "encoding/json" + "io" + "net/http" + + "git.kapelle.org/niklas/s3share/internal/client" + "github.com/gorilla/mux" + "github.com/sirupsen/logrus" +) + +type createShare struct { + Key string `json:"key"` +} + +func StartWebserver(addr string, client client.Client) error { + r := mux.NewRouter() + + r.HandleFunc("/{slug:[a-zA-Z0-9]{6}}", func(w http.ResponseWriter, r *http.Request) { + + }) + + r.HandleFunc("/s/{slug:[a-zA-Z0-9]{6}}", func(w http.ResponseWriter, r *http.Request) { + vars := mux.Vars(r) + + share, err := client.GetShare(r.Context(), vars["slug"]) + if err != nil { + http.Error(w, err.Error(), http.StatusInternalServerError) + return + } + + if share == nil { + http.NotFound(w, r) + return + } + + obj, err := client.GetObjectFromShare(r.Context(), share) + if err != nil { + http.Error(w, err.Error(), http.StatusInternalServerError) + return + } + + _, err = io.Copy(w, obj) + + if err != nil { + w.WriteHeader(http.StatusInternalServerError) + return + } + }) + + r.HandleFunc("/api/share", func(w http.ResponseWriter, r *http.Request) { + + }).Methods("GET") + + r.HandleFunc("/api/share", func(w http.ResponseWriter, r *http.Request) { + // TODO: check auth + + var shareParams createShare + err := json.NewDecoder(r.Body).Decode(&shareParams) + if err != nil { + http.Error(w, err.Error(), http.StatusBadRequest) + return + } + + share, err := client.CreateShare(r.Context(), shareParams.Key) + + if err != nil { + http.Error(w, err.Error(), http.StatusInternalServerError) + return + } + + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(http.StatusOK) + json.NewEncoder(w).Encode(share) + + }).Methods("POST") + + r.HandleFunc("/api/share", func(w http.ResponseWriter, r *http.Request) { + + }).Methods("DELETE") + + logrus.Info("Starting webserver") + return http.ListenAndServe(addr, r) +}