Last active
October 8, 2018 02:54
-
-
Save chrisgoffinet/571f0380e31b92518bd1cd24ca91a5c7 to your computer and use it in GitHub Desktop.
Upload to S3 using PerRequestContext
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| package main | |
| import ( | |
| "context" | |
| "fmt" | |
| "log" | |
| "os" | |
| "time" | |
| "github.com/aws/aws-sdk-go-v2/aws" | |
| "github.com/aws/aws-sdk-go-v2/aws/external" | |
| "github.com/aws/aws-sdk-go-v2/service/s3" | |
| "github.com/aws/aws-sdk-go-v2/service/s3/s3manager" | |
| ) | |
| func main() { | |
| bucket := "mybucket" | |
| cfg, err := external.LoadDefaultAWSConfig() | |
| if err != nil { | |
| log.Fatal("unable to load SDK config, " + err.Error()) | |
| } | |
| cfg.Region = "us-east-2" | |
| cfg.Retryer = aws.DefaultRetryer{NumMaxRetries: 10} | |
| uploader := s3manager.NewUploader(cfg) | |
| svc := s3.New(cfg) | |
| req := svc.HeadBucketRequest(&s3.HeadBucketInput{Bucket: &bucket}) | |
| _, err = req.Send() | |
| if err != nil { | |
| log.Fatal(err) | |
| } | |
| f, err := os.Open("big-file") | |
| if err != nil { | |
| log.Fatal(err) | |
| } | |
| // details of file to upload | |
| params := &s3manager.UploadInput{ | |
| Bucket: aws.String(bucket), | |
| Body: f, | |
| Key: aws.String("big-file"), | |
| } | |
| // upload file | |
| ctx := context.Background() | |
| resp, err := uploader.UploadWithContext(ctx, params, func(u *s3manager.Uploader) { | |
| u.MaxUploadParts = 10000 // set to maximum allowed by s3 | |
| u.PartSize = 5 * 1024 * 1024 // 5MB | |
| u.RequestOptions = append(u.RequestOptions, PerRequestContext()) | |
| }) | |
| fmt.Println(resp) | |
| fmt.Println(err) | |
| } | |
| // PerRequestContext sets a custom context per request to S3 | |
| // | |
| // The DefaultUploadMgr runs with concurrency of 5 goroutines | |
| // This helps when you might have slow upload but you don't want to set a context | |
| // that times out for the entire upload, it's easier to reason about per-part | |
| func PerRequestContext() aws.Option { | |
| return func(r *aws.Request) { | |
| ctx, cancel := context.WithTimeout(context.Background(), time.Minute) | |
| r.Handlers.Complete.PushBack(func(f *aws.Request) { | |
| cancel() | |
| }) | |
| r.SetContext(ctx) | |
| } | |
| } |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment