|
Packit |
63bb0d |
package s3manager
|
|
Packit |
63bb0d |
|
|
Packit |
63bb0d |
import (
|
|
Packit |
63bb0d |
"bytes"
|
|
Packit |
63bb0d |
"fmt"
|
|
Packit |
63bb0d |
"io"
|
|
Packit |
63bb0d |
|
|
Packit |
63bb0d |
"github.com/aws/aws-sdk-go/aws"
|
|
Packit |
63bb0d |
"github.com/aws/aws-sdk-go/aws/awserr"
|
|
Packit |
63bb0d |
"github.com/aws/aws-sdk-go/aws/client"
|
|
Packit |
63bb0d |
"github.com/aws/aws-sdk-go/aws/request"
|
|
Packit |
63bb0d |
"github.com/aws/aws-sdk-go/service/s3"
|
|
Packit |
63bb0d |
"github.com/aws/aws-sdk-go/service/s3/s3iface"
|
|
Packit |
63bb0d |
)
|
|
Packit |
63bb0d |
|
|
Packit |
63bb0d |
const (
|
|
Packit |
63bb0d |
// DefaultBatchSize is the batch size we initialize when constructing a batch delete client.
|
|
Packit |
63bb0d |
// This value is used when calling DeleteObjects. This represents how many objects to delete
|
|
Packit |
63bb0d |
// per DeleteObjects call.
|
|
Packit |
63bb0d |
DefaultBatchSize = 100
|
|
Packit |
63bb0d |
)
|
|
Packit |
63bb0d |
|
|
Packit |
63bb0d |
// BatchError will contain the key and bucket of the object that failed to
|
|
Packit |
63bb0d |
// either upload or download.
|
|
Packit |
63bb0d |
type BatchError struct {
|
|
Packit |
63bb0d |
Errors Errors
|
|
Packit |
63bb0d |
code string
|
|
Packit |
63bb0d |
message string
|
|
Packit |
63bb0d |
}
|
|
Packit |
63bb0d |
|
|
Packit |
63bb0d |
// Errors is a typed alias for a slice of errors to satisfy the error
|
|
Packit |
63bb0d |
// interface.
|
|
Packit |
63bb0d |
type Errors []Error
|
|
Packit |
63bb0d |
|
|
Packit |
63bb0d |
func (errs Errors) Error() string {
|
|
Packit |
63bb0d |
buf := bytes.NewBuffer(nil)
|
|
Packit |
63bb0d |
for i, err := range errs {
|
|
Packit |
63bb0d |
buf.WriteString(err.Error())
|
|
Packit |
63bb0d |
if i+1 < len(errs) {
|
|
Packit |
63bb0d |
buf.WriteString("\n")
|
|
Packit |
63bb0d |
}
|
|
Packit |
63bb0d |
}
|
|
Packit |
63bb0d |
return buf.String()
|
|
Packit |
63bb0d |
}
|
|
Packit |
63bb0d |
|
|
Packit |
63bb0d |
// Error will contain the original error, bucket, and key of the operation that failed
|
|
Packit |
63bb0d |
// during batch operations.
|
|
Packit |
63bb0d |
type Error struct {
|
|
Packit |
63bb0d |
OrigErr error
|
|
Packit |
63bb0d |
Bucket *string
|
|
Packit |
63bb0d |
Key *string
|
|
Packit |
63bb0d |
}
|
|
Packit |
63bb0d |
|
|
Packit |
63bb0d |
func newError(err error, bucket, key *string) Error {
|
|
Packit |
63bb0d |
return Error{
|
|
Packit |
63bb0d |
err,
|
|
Packit |
63bb0d |
bucket,
|
|
Packit |
63bb0d |
key,
|
|
Packit |
63bb0d |
}
|
|
Packit |
63bb0d |
}
|
|
Packit |
63bb0d |
|
|
Packit |
63bb0d |
func (err *Error) Error() string {
|
|
Packit |
63bb0d |
origErr := ""
|
|
Packit |
63bb0d |
if err.OrigErr != nil {
|
|
Packit |
63bb0d |
origErr = ":\n" + err.OrigErr.Error()
|
|
Packit |
63bb0d |
}
|
|
Packit |
63bb0d |
return fmt.Sprintf("failed to perform batch operation on %q to %q%s",
|
|
Packit |
63bb0d |
aws.StringValue(err.Key),
|
|
Packit |
63bb0d |
aws.StringValue(err.Bucket),
|
|
Packit |
63bb0d |
origErr,
|
|
Packit |
63bb0d |
)
|
|
Packit |
63bb0d |
}
|
|
Packit |
63bb0d |
|
|
Packit |
63bb0d |
// NewBatchError will return a BatchError that satisfies the awserr.Error interface.
|
|
Packit |
63bb0d |
func NewBatchError(code, message string, err []Error) awserr.Error {
|
|
Packit |
63bb0d |
return &BatchError{
|
|
Packit |
63bb0d |
Errors: err,
|
|
Packit |
63bb0d |
code: code,
|
|
Packit |
63bb0d |
message: message,
|
|
Packit |
63bb0d |
}
|
|
Packit |
63bb0d |
}
|
|
Packit |
63bb0d |
|
|
Packit |
63bb0d |
// Code will return the code associated with the batch error.
|
|
Packit |
63bb0d |
func (err *BatchError) Code() string {
|
|
Packit |
63bb0d |
return err.code
|
|
Packit |
63bb0d |
}
|
|
Packit |
63bb0d |
|
|
Packit |
63bb0d |
// Message will return the message associated with the batch error.
|
|
Packit |
63bb0d |
func (err *BatchError) Message() string {
|
|
Packit |
63bb0d |
return err.message
|
|
Packit |
63bb0d |
}
|
|
Packit |
63bb0d |
|
|
Packit |
63bb0d |
func (err *BatchError) Error() string {
|
|
Packit |
63bb0d |
return awserr.SprintError(err.Code(), err.Message(), "", err.Errors)
|
|
Packit |
63bb0d |
}
|
|
Packit |
63bb0d |
|
|
Packit |
63bb0d |
// OrigErr will return the original error. Which, in this case, will always be nil
|
|
Packit |
63bb0d |
// for batched operations.
|
|
Packit |
63bb0d |
func (err *BatchError) OrigErr() error {
|
|
Packit |
63bb0d |
return err.Errors
|
|
Packit |
63bb0d |
}
|
|
Packit |
63bb0d |
|
|
Packit |
63bb0d |
// BatchDeleteIterator is an interface that uses the scanner pattern to
|
|
Packit |
63bb0d |
// iterate through what needs to be deleted.
|
|
Packit |
63bb0d |
type BatchDeleteIterator interface {
|
|
Packit |
63bb0d |
Next() bool
|
|
Packit |
63bb0d |
Err() error
|
|
Packit |
63bb0d |
DeleteObject() BatchDeleteObject
|
|
Packit |
63bb0d |
}
|
|
Packit |
63bb0d |
|
|
Packit |
63bb0d |
// DeleteListIterator is an alternative iterator for the BatchDelete client. This will
|
|
Packit |
63bb0d |
// iterate through a list of objects and delete the objects.
|
|
Packit |
63bb0d |
//
|
|
Packit |
63bb0d |
// Example:
|
|
Packit |
63bb0d |
// iter := &s3manager.DeleteListIterator{
|
|
Packit |
63bb0d |
// Client: svc,
|
|
Packit |
63bb0d |
// Input: &s3.ListObjectsInput{
|
|
Packit |
63bb0d |
// Bucket: aws.String("bucket"),
|
|
Packit |
63bb0d |
// MaxKeys: aws.Int64(5),
|
|
Packit |
63bb0d |
// },
|
|
Packit |
63bb0d |
// Paginator: request.Pagination{
|
|
Packit |
63bb0d |
// NewRequest: func() (*request.Request, error) {
|
|
Packit |
63bb0d |
// var inCpy *ListObjectsInput
|
|
Packit |
63bb0d |
// if input != nil {
|
|
Packit |
63bb0d |
// tmp := *input
|
|
Packit |
63bb0d |
// inCpy = &tmp
|
|
Packit |
63bb0d |
// }
|
|
Packit |
63bb0d |
// req, _ := c.ListObjectsRequest(inCpy)
|
|
Packit |
63bb0d |
// return req, nil
|
|
Packit |
63bb0d |
// },
|
|
Packit |
63bb0d |
// },
|
|
Packit |
63bb0d |
// }
|
|
Packit |
63bb0d |
//
|
|
Packit |
63bb0d |
// batcher := s3manager.NewBatchDeleteWithClient(svc)
|
|
Packit |
63bb0d |
// if err := batcher.Delete(aws.BackgroundContext(), iter); err != nil {
|
|
Packit |
63bb0d |
// return err
|
|
Packit |
63bb0d |
// }
|
|
Packit |
63bb0d |
type DeleteListIterator struct {
|
|
Packit |
63bb0d |
Bucket *string
|
|
Packit |
63bb0d |
Paginator request.Pagination
|
|
Packit |
63bb0d |
objects []*s3.Object
|
|
Packit |
63bb0d |
}
|
|
Packit |
63bb0d |
|
|
Packit |
63bb0d |
// NewDeleteListIterator will return a new DeleteListIterator.
|
|
Packit |
63bb0d |
func NewDeleteListIterator(svc s3iface.S3API, input *s3.ListObjectsInput, opts ...func(*DeleteListIterator)) BatchDeleteIterator {
|
|
Packit |
63bb0d |
iter := &DeleteListIterator{
|
|
Packit |
63bb0d |
Bucket: input.Bucket,
|
|
Packit |
63bb0d |
Paginator: request.Pagination{
|
|
Packit |
63bb0d |
NewRequest: func() (*request.Request, error) {
|
|
Packit |
63bb0d |
var inCpy *s3.ListObjectsInput
|
|
Packit |
63bb0d |
if input != nil {
|
|
Packit |
63bb0d |
tmp := *input
|
|
Packit |
63bb0d |
inCpy = &tmp
|
|
Packit |
63bb0d |
}
|
|
Packit |
63bb0d |
req, _ := svc.ListObjectsRequest(inCpy)
|
|
Packit |
63bb0d |
return req, nil
|
|
Packit |
63bb0d |
},
|
|
Packit |
63bb0d |
},
|
|
Packit |
63bb0d |
}
|
|
Packit |
63bb0d |
|
|
Packit |
63bb0d |
for _, opt := range opts {
|
|
Packit |
63bb0d |
opt(iter)
|
|
Packit |
63bb0d |
}
|
|
Packit |
63bb0d |
return iter
|
|
Packit |
63bb0d |
}
|
|
Packit |
63bb0d |
|
|
Packit |
63bb0d |
// Next will use the S3API client to iterate through a list of objects.
|
|
Packit |
63bb0d |
func (iter *DeleteListIterator) Next() bool {
|
|
Packit |
63bb0d |
if len(iter.objects) > 0 {
|
|
Packit |
63bb0d |
iter.objects = iter.objects[1:]
|
|
Packit |
63bb0d |
}
|
|
Packit |
63bb0d |
|
|
Packit |
63bb0d |
if len(iter.objects) == 0 && iter.Paginator.Next() {
|
|
Packit |
63bb0d |
iter.objects = iter.Paginator.Page().(*s3.ListObjectsOutput).Contents
|
|
Packit |
63bb0d |
}
|
|
Packit |
63bb0d |
|
|
Packit |
63bb0d |
return len(iter.objects) > 0
|
|
Packit |
63bb0d |
}
|
|
Packit |
63bb0d |
|
|
Packit |
63bb0d |
// Err will return the last known error from Next.
|
|
Packit |
63bb0d |
func (iter *DeleteListIterator) Err() error {
|
|
Packit |
63bb0d |
return iter.Paginator.Err()
|
|
Packit |
63bb0d |
}
|
|
Packit |
63bb0d |
|
|
Packit |
63bb0d |
// DeleteObject will return the current object to be deleted.
|
|
Packit |
63bb0d |
func (iter *DeleteListIterator) DeleteObject() BatchDeleteObject {
|
|
Packit |
63bb0d |
return BatchDeleteObject{
|
|
Packit |
63bb0d |
Object: &s3.DeleteObjectInput{
|
|
Packit |
63bb0d |
Bucket: iter.Bucket,
|
|
Packit |
63bb0d |
Key: iter.objects[0].Key,
|
|
Packit |
63bb0d |
},
|
|
Packit |
63bb0d |
}
|
|
Packit |
63bb0d |
}
|
|
Packit |
63bb0d |
|
|
Packit |
63bb0d |
// BatchDelete will use the s3 package's service client to perform a batch
|
|
Packit |
63bb0d |
// delete.
|
|
Packit |
63bb0d |
type BatchDelete struct {
|
|
Packit |
63bb0d |
Client s3iface.S3API
|
|
Packit |
63bb0d |
BatchSize int
|
|
Packit |
63bb0d |
}
|
|
Packit |
63bb0d |
|
|
Packit |
63bb0d |
// NewBatchDeleteWithClient will return a new delete client that can delete a batched amount of
|
|
Packit |
63bb0d |
// objects.
|
|
Packit |
63bb0d |
//
|
|
Packit |
63bb0d |
// Example:
|
|
Packit |
63bb0d |
// batcher := s3manager.NewBatchDeleteWithClient(client, size)
|
|
Packit |
63bb0d |
//
|
|
Packit |
63bb0d |
// objects := []BatchDeleteObject{
|
|
Packit |
63bb0d |
// {
|
|
Packit |
63bb0d |
// Object: &s3.DeleteObjectInput {
|
|
Packit |
63bb0d |
// Key: aws.String("key"),
|
|
Packit |
63bb0d |
// Bucket: aws.String("bucket"),
|
|
Packit |
63bb0d |
// },
|
|
Packit |
63bb0d |
// },
|
|
Packit |
63bb0d |
// }
|
|
Packit |
63bb0d |
//
|
|
Packit |
63bb0d |
// if err := batcher.Delete(aws.BackgroundContext(), &s3manager.DeleteObjectsIterator{
|
|
Packit |
63bb0d |
// Objects: objects,
|
|
Packit |
63bb0d |
// }); err != nil {
|
|
Packit |
63bb0d |
// return err
|
|
Packit |
63bb0d |
// }
|
|
Packit |
63bb0d |
func NewBatchDeleteWithClient(client s3iface.S3API, options ...func(*BatchDelete)) *BatchDelete {
|
|
Packit |
63bb0d |
svc := &BatchDelete{
|
|
Packit |
63bb0d |
Client: client,
|
|
Packit |
63bb0d |
BatchSize: DefaultBatchSize,
|
|
Packit |
63bb0d |
}
|
|
Packit |
63bb0d |
|
|
Packit |
63bb0d |
for _, opt := range options {
|
|
Packit |
63bb0d |
opt(svc)
|
|
Packit |
63bb0d |
}
|
|
Packit |
63bb0d |
|
|
Packit |
63bb0d |
return svc
|
|
Packit |
63bb0d |
}
|
|
Packit |
63bb0d |
|
|
Packit |
63bb0d |
// NewBatchDelete will return a new delete client that can delete a batched amount of
|
|
Packit |
63bb0d |
// objects.
|
|
Packit |
63bb0d |
//
|
|
Packit |
63bb0d |
// Example:
|
|
Packit |
63bb0d |
// batcher := s3manager.NewBatchDelete(sess, size)
|
|
Packit |
63bb0d |
//
|
|
Packit |
63bb0d |
// objects := []BatchDeleteObject{
|
|
Packit |
63bb0d |
// {
|
|
Packit |
63bb0d |
// Object: &s3.DeleteObjectInput {
|
|
Packit |
63bb0d |
// Key: aws.String("key"),
|
|
Packit |
63bb0d |
// Bucket: aws.String("bucket"),
|
|
Packit |
63bb0d |
// },
|
|
Packit |
63bb0d |
// },
|
|
Packit |
63bb0d |
// }
|
|
Packit |
63bb0d |
//
|
|
Packit |
63bb0d |
// if err := batcher.Delete(aws.BackgroundContext(), &s3manager.DeleteObjectsIterator{
|
|
Packit |
63bb0d |
// Objects: objects,
|
|
Packit |
63bb0d |
// }); err != nil {
|
|
Packit |
63bb0d |
// return err
|
|
Packit |
63bb0d |
// }
|
|
Packit |
63bb0d |
func NewBatchDelete(c client.ConfigProvider, options ...func(*BatchDelete)) *BatchDelete {
|
|
Packit |
63bb0d |
client := s3.New(c)
|
|
Packit |
63bb0d |
return NewBatchDeleteWithClient(client, options...)
|
|
Packit |
63bb0d |
}
|
|
Packit |
63bb0d |
|
|
Packit |
63bb0d |
// BatchDeleteObject is a wrapper object for calling the batch delete operation.
|
|
Packit |
63bb0d |
type BatchDeleteObject struct {
|
|
Packit |
63bb0d |
Object *s3.DeleteObjectInput
|
|
Packit |
63bb0d |
// After will run after each iteration during the batch process. This function will
|
|
Packit |
63bb0d |
// be executed whether or not the request was successful.
|
|
Packit |
63bb0d |
After func() error
|
|
Packit |
63bb0d |
}
|
|
Packit |
63bb0d |
|
|
Packit |
63bb0d |
// DeleteObjectsIterator is an interface that uses the scanner pattern to iterate
|
|
Packit |
63bb0d |
// through a series of objects to be deleted.
|
|
Packit |
63bb0d |
type DeleteObjectsIterator struct {
|
|
Packit |
63bb0d |
Objects []BatchDeleteObject
|
|
Packit |
63bb0d |
index int
|
|
Packit |
63bb0d |
inc bool
|
|
Packit |
63bb0d |
}
|
|
Packit |
63bb0d |
|
|
Packit |
63bb0d |
// Next will increment the default iterator's index and ensure that there
|
|
Packit |
63bb0d |
// is another object to iterator to.
|
|
Packit |
63bb0d |
func (iter *DeleteObjectsIterator) Next() bool {
|
|
Packit |
63bb0d |
if iter.inc {
|
|
Packit |
63bb0d |
iter.index++
|
|
Packit |
63bb0d |
} else {
|
|
Packit |
63bb0d |
iter.inc = true
|
|
Packit |
63bb0d |
}
|
|
Packit |
63bb0d |
return iter.index < len(iter.Objects)
|
|
Packit |
63bb0d |
}
|
|
Packit |
63bb0d |
|
|
Packit |
63bb0d |
// Err will return an error. Since this is just used to satisfy the BatchDeleteIterator interface
|
|
Packit |
63bb0d |
// this will only return nil.
|
|
Packit |
63bb0d |
func (iter *DeleteObjectsIterator) Err() error {
|
|
Packit |
63bb0d |
return nil
|
|
Packit |
63bb0d |
}
|
|
Packit |
63bb0d |
|
|
Packit |
63bb0d |
// DeleteObject will return the BatchDeleteObject at the current batched index.
|
|
Packit |
63bb0d |
func (iter *DeleteObjectsIterator) DeleteObject() BatchDeleteObject {
|
|
Packit |
63bb0d |
object := iter.Objects[iter.index]
|
|
Packit |
63bb0d |
return object
|
|
Packit |
63bb0d |
}
|
|
Packit |
63bb0d |
|
|
Packit |
63bb0d |
// Delete will use the iterator to queue up objects that need to be deleted.
|
|
Packit |
63bb0d |
// Once the batch size is met, this will call the deleteBatch function.
|
|
Packit |
63bb0d |
func (d *BatchDelete) Delete(ctx aws.Context, iter BatchDeleteIterator) error {
|
|
Packit |
63bb0d |
var errs []Error
|
|
Packit |
63bb0d |
objects := []BatchDeleteObject{}
|
|
Packit |
63bb0d |
var input *s3.DeleteObjectsInput
|
|
Packit |
63bb0d |
|
|
Packit |
63bb0d |
for iter.Next() {
|
|
Packit |
63bb0d |
o := iter.DeleteObject()
|
|
Packit |
63bb0d |
|
|
Packit |
63bb0d |
if input == nil {
|
|
Packit |
63bb0d |
input = initDeleteObjectsInput(o.Object)
|
|
Packit |
63bb0d |
}
|
|
Packit |
63bb0d |
|
|
Packit |
63bb0d |
parity := hasParity(input, o)
|
|
Packit |
63bb0d |
if parity {
|
|
Packit |
63bb0d |
input.Delete.Objects = append(input.Delete.Objects, &s3.ObjectIdentifier{
|
|
Packit |
63bb0d |
Key: o.Object.Key,
|
|
Packit |
63bb0d |
VersionId: o.Object.VersionId,
|
|
Packit |
63bb0d |
})
|
|
Packit |
63bb0d |
objects = append(objects, o)
|
|
Packit |
63bb0d |
}
|
|
Packit |
63bb0d |
|
|
Packit |
63bb0d |
if len(input.Delete.Objects) == d.BatchSize || !parity {
|
|
Packit |
63bb0d |
if err := deleteBatch(ctx, d, input, objects); err != nil {
|
|
Packit |
63bb0d |
errs = append(errs, err...)
|
|
Packit |
63bb0d |
}
|
|
Packit |
63bb0d |
|
|
Packit |
63bb0d |
objects = objects[:0]
|
|
Packit |
63bb0d |
input = nil
|
|
Packit |
63bb0d |
|
|
Packit |
63bb0d |
if !parity {
|
|
Packit |
63bb0d |
objects = append(objects, o)
|
|
Packit |
63bb0d |
input = initDeleteObjectsInput(o.Object)
|
|
Packit |
63bb0d |
input.Delete.Objects = append(input.Delete.Objects, &s3.ObjectIdentifier{
|
|
Packit |
63bb0d |
Key: o.Object.Key,
|
|
Packit |
63bb0d |
VersionId: o.Object.VersionId,
|
|
Packit |
63bb0d |
})
|
|
Packit |
63bb0d |
}
|
|
Packit |
63bb0d |
}
|
|
Packit |
63bb0d |
}
|
|
Packit |
63bb0d |
|
|
Packit |
63bb0d |
// iter.Next() could return false (above) plus populate iter.Err()
|
|
Packit |
63bb0d |
if iter.Err() != nil {
|
|
Packit |
63bb0d |
errs = append(errs, newError(iter.Err(), nil, nil))
|
|
Packit |
63bb0d |
}
|
|
Packit |
63bb0d |
|
|
Packit |
63bb0d |
if input != nil && len(input.Delete.Objects) > 0 {
|
|
Packit |
63bb0d |
if err := deleteBatch(ctx, d, input, objects); err != nil {
|
|
Packit |
63bb0d |
errs = append(errs, err...)
|
|
Packit |
63bb0d |
}
|
|
Packit |
63bb0d |
}
|
|
Packit |
63bb0d |
|
|
Packit |
63bb0d |
if len(errs) > 0 {
|
|
Packit |
63bb0d |
return NewBatchError("BatchedDeleteIncomplete", "some objects have failed to be deleted.", errs)
|
|
Packit |
63bb0d |
}
|
|
Packit |
63bb0d |
return nil
|
|
Packit |
63bb0d |
}
|
|
Packit |
63bb0d |
|
|
Packit |
63bb0d |
func initDeleteObjectsInput(o *s3.DeleteObjectInput) *s3.DeleteObjectsInput {
|
|
Packit |
63bb0d |
return &s3.DeleteObjectsInput{
|
|
Packit |
63bb0d |
Bucket: o.Bucket,
|
|
Packit |
63bb0d |
MFA: o.MFA,
|
|
Packit |
63bb0d |
RequestPayer: o.RequestPayer,
|
|
Packit |
63bb0d |
Delete: &s3.Delete{},
|
|
Packit |
63bb0d |
}
|
|
Packit |
63bb0d |
}
|
|
Packit |
63bb0d |
|
|
Packit |
63bb0d |
const (
|
|
Packit |
63bb0d |
// ErrDeleteBatchFailCode represents an error code which will be returned
|
|
Packit |
63bb0d |
// only when DeleteObjects.Errors has an error that does not contain a code.
|
|
Packit |
63bb0d |
ErrDeleteBatchFailCode = "DeleteBatchError"
|
|
Packit |
63bb0d |
errDefaultDeleteBatchMessage = "failed to delete"
|
|
Packit |
63bb0d |
)
|
|
Packit |
63bb0d |
|
|
Packit |
63bb0d |
// deleteBatch will delete a batch of items in the objects parameters.
|
|
Packit |
63bb0d |
func deleteBatch(ctx aws.Context, d *BatchDelete, input *s3.DeleteObjectsInput, objects []BatchDeleteObject) []Error {
|
|
Packit |
63bb0d |
errs := []Error{}
|
|
Packit |
63bb0d |
|
|
Packit |
63bb0d |
if result, err := d.Client.DeleteObjectsWithContext(ctx, input); err != nil {
|
|
Packit |
63bb0d |
for i := 0; i < len(input.Delete.Objects); i++ {
|
|
Packit |
63bb0d |
errs = append(errs, newError(err, input.Bucket, input.Delete.Objects[i].Key))
|
|
Packit |
63bb0d |
}
|
|
Packit |
63bb0d |
} else if len(result.Errors) > 0 {
|
|
Packit |
63bb0d |
for i := 0; i < len(result.Errors); i++ {
|
|
Packit |
63bb0d |
code := ErrDeleteBatchFailCode
|
|
Packit |
63bb0d |
msg := errDefaultDeleteBatchMessage
|
|
Packit |
63bb0d |
if result.Errors[i].Message != nil {
|
|
Packit |
63bb0d |
msg = *result.Errors[i].Message
|
|
Packit |
63bb0d |
}
|
|
Packit |
63bb0d |
if result.Errors[i].Code != nil {
|
|
Packit |
63bb0d |
code = *result.Errors[i].Code
|
|
Packit |
63bb0d |
}
|
|
Packit |
63bb0d |
|
|
Packit |
63bb0d |
errs = append(errs, newError(awserr.New(code, msg, err), input.Bucket, result.Errors[i].Key))
|
|
Packit |
63bb0d |
}
|
|
Packit |
63bb0d |
}
|
|
Packit |
63bb0d |
for _, object := range objects {
|
|
Packit |
63bb0d |
if object.After == nil {
|
|
Packit |
63bb0d |
continue
|
|
Packit |
63bb0d |
}
|
|
Packit |
63bb0d |
if err := object.After(); err != nil {
|
|
Packit |
63bb0d |
errs = append(errs, newError(err, object.Object.Bucket, object.Object.Key))
|
|
Packit |
63bb0d |
}
|
|
Packit |
63bb0d |
}
|
|
Packit |
63bb0d |
|
|
Packit |
63bb0d |
return errs
|
|
Packit |
63bb0d |
}
|
|
Packit |
63bb0d |
|
|
Packit |
63bb0d |
func hasParity(o1 *s3.DeleteObjectsInput, o2 BatchDeleteObject) bool {
|
|
Packit |
63bb0d |
if o1.Bucket != nil && o2.Object.Bucket != nil {
|
|
Packit |
63bb0d |
if *o1.Bucket != *o2.Object.Bucket {
|
|
Packit |
63bb0d |
return false
|
|
Packit |
63bb0d |
}
|
|
Packit |
63bb0d |
} else if o1.Bucket != o2.Object.Bucket {
|
|
Packit |
63bb0d |
return false
|
|
Packit |
63bb0d |
}
|
|
Packit |
63bb0d |
|
|
Packit |
63bb0d |
if o1.MFA != nil && o2.Object.MFA != nil {
|
|
Packit |
63bb0d |
if *o1.MFA != *o2.Object.MFA {
|
|
Packit |
63bb0d |
return false
|
|
Packit |
63bb0d |
}
|
|
Packit |
63bb0d |
} else if o1.MFA != o2.Object.MFA {
|
|
Packit |
63bb0d |
return false
|
|
Packit |
63bb0d |
}
|
|
Packit |
63bb0d |
|
|
Packit |
63bb0d |
if o1.RequestPayer != nil && o2.Object.RequestPayer != nil {
|
|
Packit |
63bb0d |
if *o1.RequestPayer != *o2.Object.RequestPayer {
|
|
Packit |
63bb0d |
return false
|
|
Packit |
63bb0d |
}
|
|
Packit |
63bb0d |
} else if o1.RequestPayer != o2.Object.RequestPayer {
|
|
Packit |
63bb0d |
return false
|
|
Packit |
63bb0d |
}
|
|
Packit |
63bb0d |
|
|
Packit |
63bb0d |
return true
|
|
Packit |
63bb0d |
}
|
|
Packit |
63bb0d |
|
|
Packit |
63bb0d |
// BatchDownloadIterator is an interface that uses the scanner pattern to iterate
|
|
Packit |
63bb0d |
// through a series of objects to be downloaded.
|
|
Packit |
63bb0d |
type BatchDownloadIterator interface {
|
|
Packit |
63bb0d |
Next() bool
|
|
Packit |
63bb0d |
Err() error
|
|
Packit |
63bb0d |
DownloadObject() BatchDownloadObject
|
|
Packit |
63bb0d |
}
|
|
Packit |
63bb0d |
|
|
Packit |
63bb0d |
// BatchDownloadObject contains all necessary information to run a batch operation once.
|
|
Packit |
63bb0d |
type BatchDownloadObject struct {
|
|
Packit |
63bb0d |
Object *s3.GetObjectInput
|
|
Packit |
63bb0d |
Writer io.WriterAt
|
|
Packit |
63bb0d |
// After will run after each iteration during the batch process. This function will
|
|
Packit |
63bb0d |
// be executed whether or not the request was successful.
|
|
Packit |
63bb0d |
After func() error
|
|
Packit |
63bb0d |
}
|
|
Packit |
63bb0d |
|
|
Packit |
63bb0d |
// DownloadObjectsIterator implements the BatchDownloadIterator interface and allows for batched
|
|
Packit |
63bb0d |
// download of objects.
|
|
Packit |
63bb0d |
type DownloadObjectsIterator struct {
|
|
Packit |
63bb0d |
Objects []BatchDownloadObject
|
|
Packit |
63bb0d |
index int
|
|
Packit |
63bb0d |
inc bool
|
|
Packit |
63bb0d |
}
|
|
Packit |
63bb0d |
|
|
Packit |
63bb0d |
// Next will increment the default iterator's index and ensure that there
|
|
Packit |
63bb0d |
// is another object to iterator to.
|
|
Packit |
63bb0d |
func (batcher *DownloadObjectsIterator) Next() bool {
|
|
Packit |
63bb0d |
if batcher.inc {
|
|
Packit |
63bb0d |
batcher.index++
|
|
Packit |
63bb0d |
} else {
|
|
Packit |
63bb0d |
batcher.inc = true
|
|
Packit |
63bb0d |
}
|
|
Packit |
63bb0d |
return batcher.index < len(batcher.Objects)
|
|
Packit |
63bb0d |
}
|
|
Packit |
63bb0d |
|
|
Packit |
63bb0d |
// DownloadObject will return the BatchDownloadObject at the current batched index.
|
|
Packit |
63bb0d |
func (batcher *DownloadObjectsIterator) DownloadObject() BatchDownloadObject {
|
|
Packit |
63bb0d |
object := batcher.Objects[batcher.index]
|
|
Packit |
63bb0d |
return object
|
|
Packit |
63bb0d |
}
|
|
Packit |
63bb0d |
|
|
Packit |
63bb0d |
// Err will return an error. Since this is just used to satisfy the BatchDeleteIterator interface
|
|
Packit |
63bb0d |
// this will only return nil.
|
|
Packit |
63bb0d |
func (batcher *DownloadObjectsIterator) Err() error {
|
|
Packit |
63bb0d |
return nil
|
|
Packit |
63bb0d |
}
|
|
Packit |
63bb0d |
|
|
Packit |
63bb0d |
// BatchUploadIterator is an interface that uses the scanner pattern to
|
|
Packit |
63bb0d |
// iterate through what needs to be uploaded.
|
|
Packit |
63bb0d |
type BatchUploadIterator interface {
|
|
Packit |
63bb0d |
Next() bool
|
|
Packit |
63bb0d |
Err() error
|
|
Packit |
63bb0d |
UploadObject() BatchUploadObject
|
|
Packit |
63bb0d |
}
|
|
Packit |
63bb0d |
|
|
Packit |
63bb0d |
// UploadObjectsIterator implements the BatchUploadIterator interface and allows for batched
|
|
Packit |
63bb0d |
// upload of objects.
|
|
Packit |
63bb0d |
type UploadObjectsIterator struct {
|
|
Packit |
63bb0d |
Objects []BatchUploadObject
|
|
Packit |
63bb0d |
index int
|
|
Packit |
63bb0d |
inc bool
|
|
Packit |
63bb0d |
}
|
|
Packit |
63bb0d |
|
|
Packit |
63bb0d |
// Next will increment the default iterator's index and ensure that there
|
|
Packit |
63bb0d |
// is another object to iterator to.
|
|
Packit |
63bb0d |
func (batcher *UploadObjectsIterator) Next() bool {
|
|
Packit |
63bb0d |
if batcher.inc {
|
|
Packit |
63bb0d |
batcher.index++
|
|
Packit |
63bb0d |
} else {
|
|
Packit |
63bb0d |
batcher.inc = true
|
|
Packit |
63bb0d |
}
|
|
Packit |
63bb0d |
return batcher.index < len(batcher.Objects)
|
|
Packit |
63bb0d |
}
|
|
Packit |
63bb0d |
|
|
Packit |
63bb0d |
// Err will return an error. Since this is just used to satisfy the BatchUploadIterator interface
|
|
Packit |
63bb0d |
// this will only return nil.
|
|
Packit |
63bb0d |
func (batcher *UploadObjectsIterator) Err() error {
|
|
Packit |
63bb0d |
return nil
|
|
Packit |
63bb0d |
}
|
|
Packit |
63bb0d |
|
|
Packit |
63bb0d |
// UploadObject will return the BatchUploadObject at the current batched index.
|
|
Packit |
63bb0d |
func (batcher *UploadObjectsIterator) UploadObject() BatchUploadObject {
|
|
Packit |
63bb0d |
object := batcher.Objects[batcher.index]
|
|
Packit |
63bb0d |
return object
|
|
Packit |
63bb0d |
}
|
|
Packit |
63bb0d |
|
|
Packit |
63bb0d |
// BatchUploadObject contains all necessary information to run a batch operation once.
|
|
Packit |
63bb0d |
type BatchUploadObject struct {
|
|
Packit |
63bb0d |
Object *UploadInput
|
|
Packit |
63bb0d |
// After will run after each iteration during the batch process. This function will
|
|
Packit |
63bb0d |
// be executed whether or not the request was successful.
|
|
Packit |
63bb0d |
After func() error
|
|
Packit |
63bb0d |
}
|