Last active
August 20, 2022 15:40
-
-
Save jszwec/f314aa464d1643c55bb84c689368aee8 to your computer and use it in GitHub Desktop.
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
package main | |
import ( | |
"bytes" | |
"crypto/tls" | |
"flag" | |
"fmt" | |
"io" | |
"io/ioutil" | |
"log" | |
"net/http" | |
"os" | |
"github.com/aws/aws-sdk-go/aws" | |
"github.com/aws/aws-sdk-go/aws/credentials" | |
"github.com/aws/aws-sdk-go/aws/session" | |
"github.com/aws/aws-sdk-go/service/s3" | |
"github.com/aws/aws-sdk-go/service/s3/s3iface" | |
"github.com/aws/aws-sdk-go/service/s3/s3manager" | |
"github.com/jszwec/s3fs" | |
) | |
var ( | |
endpoint = flag.String("endpoint", "localhost:4566", "s3 endpoint") | |
bucket = flag.String("bucket", "test-github.com-jszwec-s3fs", "bucket name") | |
skipVerify = flag.Bool("skip-verify", true, "http insecure skip verify") | |
) | |
var ( | |
accessKeyID = envDefault("S3FS_TEST_AWS_ACCESS_KEY_ID", "1") | |
secretKey = envDefault("S3FS_TEST_AWS_SECRET_ACCESS_KEY", "1") | |
region = envDefault("S3FS_TEST_AWS_REGION", "us-east-1") | |
) | |
func main() { | |
flag.Parse() | |
var osfsys osFS | |
s3fsys, err := newS3FS(*bucket) | |
if err != nil { | |
panic("failed to set up s3 client: " + err.Error()) | |
} | |
var ( | |
old = []byte("original content") | |
new = []byte("w00t") | |
) | |
for _, tt := range []struct { | |
desc string | |
fsys FS | |
}{ | |
{"os file system", osfsys}, | |
{"s3 fsys", s3fsys}, | |
} { | |
if err := doTest(tt.fsys, old, new); err != nil { | |
log.Printf("%s failed: %v", tt.desc, err) | |
continue | |
} | |
log.Printf("%s ok", tt.desc) | |
} | |
} | |
func doTest(fsys FS, old, new []byte) error { | |
const filename = "s3fs-test.txt" | |
if err := fsys.Write(filename, old); err != nil { | |
return err | |
} | |
defer fsys.Remove(filename) | |
f, err := fsys.Open(filename) | |
if err != nil { | |
return err | |
} | |
defer f.Close() | |
fseeker, ok := f.(io.ReadSeekCloser) | |
if ok { | |
if _, err := fseeker.Seek(0, io.SeekEnd); err != nil { | |
return err | |
} | |
} | |
if err := fsys.Remove(filename); err != nil { | |
return err | |
} | |
if err := fsys.Write(filename, new); err != nil { | |
return err | |
} | |
if ok { | |
if _, err := fseeker.Seek(0, io.SeekStart); err != nil { | |
return err | |
} | |
} | |
b, err := io.ReadAll(f) | |
if err != nil { | |
return err | |
} | |
if !bytes.Equal(old, b) { | |
return fmt.Errorf("expected %q; got %q", old, b) | |
} | |
return nil | |
} | |
type FS interface { | |
Open(string) (io.ReadCloser, error) | |
Write(string, []byte) error | |
Remove(string) error | |
} | |
type osFS struct{} | |
func (fs osFS) Open(path string) (io.ReadCloser, error) { | |
return os.Open(path) | |
} | |
func (fs osFS) Write(path string, data []byte) error { | |
return ioutil.WriteFile(path, data, 0655) | |
} | |
func (fs osFS) Remove(path string) error { | |
return os.RemoveAll(path) | |
} | |
type s3FS struct { | |
cl s3iface.S3API | |
bucket string | |
s3fs *s3fs.S3FS | |
} | |
func newS3FS(bucket string) (*s3FS, error) { | |
cl, err := newClient() | |
if err != nil { | |
return nil, err | |
} | |
if err := createBucket(cl, bucket); err != nil { | |
return nil, err | |
} | |
return &s3FS{ | |
cl: cl, | |
bucket: bucket, | |
s3fs: s3fs.New(cl, bucket), | |
}, nil | |
} | |
func (fs s3FS) Open(path string) (io.ReadCloser, error) { | |
return fs.s3fs.Open(path) | |
} | |
func (fs s3FS) Write(name string, data []byte) error { | |
uploader := s3manager.NewUploaderWithClient(fs.cl) | |
_, err := uploader.Upload(&s3manager.UploadInput{ | |
Body: bytes.NewReader(data), | |
Bucket: &fs.bucket, | |
Key: &name, | |
}) | |
return err | |
} | |
func (fs s3FS) Remove(path string) error { | |
_, err := fs.cl.DeleteObject(&s3.DeleteObjectInput{ | |
Bucket: &fs.bucket, | |
Key: &path, | |
}) | |
return err | |
} | |
func createBucket(cl s3iface.S3API, bucket string) error { | |
_, err := cl.CreateBucket(&s3.CreateBucketInput{ | |
Bucket: &bucket, | |
}) | |
return err | |
} | |
func newClient() (s3iface.S3API, error) { | |
cl := &http.Client{ | |
Transport: &http.Transport{ | |
TLSClientConfig: &tls.Config{ | |
InsecureSkipVerify: *skipVerify, | |
}, | |
}, | |
} | |
s, err := session.NewSession( | |
aws.NewConfig(). | |
WithEndpoint(*endpoint). | |
WithRegion(region). | |
WithS3ForcePathStyle(true). | |
WithHTTPClient(cl). | |
WithCredentials(credentials.NewStaticCredentials(accessKeyID, secretKey, "")), | |
) | |
if err != nil { | |
return nil, err | |
} | |
return s3.New(s), nil | |
} | |
func envDefault(env, def string) string { | |
if os.Getenv(env) == "" { | |
return def | |
} | |
return os.Getenv(env) | |
} |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment