mirror of
https://github.com/hanzoai/team.git
synced 2026-09-30 01:57:35 +00:00
e2bfeef066
* Fix S3 MoveFile/CopyFile failing on files larger than 5GiB S3FileBackend.MoveFile, CopyFile and DecodeFilePathIfNeeded used the minio-go CopyObject call, which performs a single server-side PUT-copy. S3 rejects copy sources larger than 5GiB, so any move/copy of a file over 5GiB fails with: The specified copy source is larger than the maximum allowable size for a copy source: 5368709120 This breaks, among other things, finalizing an `mmctl import upload` of an import archive larger than 5GiB on S3-backed installations (the upload streams fine, then the .tmp -> final rename fails). Extract the shared copy logic into a copyObject helper that picks the right S3 operation by source size: CopyObject for sources up to 5GiB, and ComposeObject (server-side multipart copy via UploadPartCopy) for larger sources, supporting objects up to 5TiB. Small files keep using a single CopyObject; only files over 5GiB pay for multipart. The size is checked explicitly rather than relying on ComposeObject's own single-copy fast path, which is only taken when the source range Start is -1, a value its input validation rejects (Start must be non-negative), so it is unreachable here. * Add unit tests for S3 copyObject copy-vs-compose routing Extract the copy-routing logic into copyObjectWithClient, which takes a small s3CopyClient interface (StatObject/CopyObject/ComposeObject) so it can be tested with a mock. The exported behavior of copyObject (and its callers MoveFile/CopyFile) is unchanged; *minio.Client satisfies the interface. Tests cover the three branches: - source > 5GiB -> ComposeObject (multipart copy), not CopyObject - source <= 5GiB -> CopyObject, not ComposeObject (incl. the boundary) - StatObject error -> propagated, neither copy is attempted * make mocks --------- Co-authored-by: wiggin77 <[email protected]> Co-authored-by: Hanzo Dev <[email protected]>
123 lines
3.6 KiB
Go
123 lines
3.6 KiB
Go
// Code generated by mockery v2.53.4. DO NOT EDIT.
|
|
|
|
// Regenerate this file using `make filestore-mocks`.
|
|
|
|
package mocks
|
|
|
|
import (
|
|
context "context"
|
|
|
|
minio "github.com/minio/minio-go/v7"
|
|
mock "github.com/stretchr/testify/mock"
|
|
)
|
|
|
|
// s3CopyClient is an autogenerated mock type for the s3CopyClient type
|
|
type s3CopyClient struct {
|
|
mock.Mock
|
|
}
|
|
|
|
// ComposeObject provides a mock function with given fields: ctx, dst, srcs
|
|
func (_m *s3CopyClient) ComposeObject(ctx context.Context, dst minio.CopyDestOptions, srcs ...minio.CopySrcOptions) (minio.UploadInfo, error) {
|
|
_va := make([]interface{}, len(srcs))
|
|
for _i := range srcs {
|
|
_va[_i] = srcs[_i]
|
|
}
|
|
var _ca []interface{}
|
|
_ca = append(_ca, ctx, dst)
|
|
_ca = append(_ca, _va...)
|
|
ret := _m.Called(_ca...)
|
|
|
|
if len(ret) == 0 {
|
|
panic("no return value specified for ComposeObject")
|
|
}
|
|
|
|
var r0 minio.UploadInfo
|
|
var r1 error
|
|
if rf, ok := ret.Get(0).(func(context.Context, minio.CopyDestOptions, ...minio.CopySrcOptions) (minio.UploadInfo, error)); ok {
|
|
return rf(ctx, dst, srcs...)
|
|
}
|
|
if rf, ok := ret.Get(0).(func(context.Context, minio.CopyDestOptions, ...minio.CopySrcOptions) minio.UploadInfo); ok {
|
|
r0 = rf(ctx, dst, srcs...)
|
|
} else {
|
|
r0 = ret.Get(0).(minio.UploadInfo)
|
|
}
|
|
|
|
if rf, ok := ret.Get(1).(func(context.Context, minio.CopyDestOptions, ...minio.CopySrcOptions) error); ok {
|
|
r1 = rf(ctx, dst, srcs...)
|
|
} else {
|
|
r1 = ret.Error(1)
|
|
}
|
|
|
|
return r0, r1
|
|
}
|
|
|
|
// CopyObject provides a mock function with given fields: ctx, dst, src
|
|
func (_m *s3CopyClient) CopyObject(ctx context.Context, dst minio.CopyDestOptions, src minio.CopySrcOptions) (minio.UploadInfo, error) {
|
|
ret := _m.Called(ctx, dst, src)
|
|
|
|
if len(ret) == 0 {
|
|
panic("no return value specified for CopyObject")
|
|
}
|
|
|
|
var r0 minio.UploadInfo
|
|
var r1 error
|
|
if rf, ok := ret.Get(0).(func(context.Context, minio.CopyDestOptions, minio.CopySrcOptions) (minio.UploadInfo, error)); ok {
|
|
return rf(ctx, dst, src)
|
|
}
|
|
if rf, ok := ret.Get(0).(func(context.Context, minio.CopyDestOptions, minio.CopySrcOptions) minio.UploadInfo); ok {
|
|
r0 = rf(ctx, dst, src)
|
|
} else {
|
|
r0 = ret.Get(0).(minio.UploadInfo)
|
|
}
|
|
|
|
if rf, ok := ret.Get(1).(func(context.Context, minio.CopyDestOptions, minio.CopySrcOptions) error); ok {
|
|
r1 = rf(ctx, dst, src)
|
|
} else {
|
|
r1 = ret.Error(1)
|
|
}
|
|
|
|
return r0, r1
|
|
}
|
|
|
|
// StatObject provides a mock function with given fields: ctx, bucketName, objectName, opts
|
|
func (_m *s3CopyClient) StatObject(ctx context.Context, bucketName string, objectName string, opts minio.GetObjectOptions) (minio.ObjectInfo, error) {
|
|
ret := _m.Called(ctx, bucketName, objectName, opts)
|
|
|
|
if len(ret) == 0 {
|
|
panic("no return value specified for StatObject")
|
|
}
|
|
|
|
var r0 minio.ObjectInfo
|
|
var r1 error
|
|
if rf, ok := ret.Get(0).(func(context.Context, string, string, minio.GetObjectOptions) (minio.ObjectInfo, error)); ok {
|
|
return rf(ctx, bucketName, objectName, opts)
|
|
}
|
|
if rf, ok := ret.Get(0).(func(context.Context, string, string, minio.GetObjectOptions) minio.ObjectInfo); ok {
|
|
r0 = rf(ctx, bucketName, objectName, opts)
|
|
} else {
|
|
r0 = ret.Get(0).(minio.ObjectInfo)
|
|
}
|
|
|
|
if rf, ok := ret.Get(1).(func(context.Context, string, string, minio.GetObjectOptions) error); ok {
|
|
r1 = rf(ctx, bucketName, objectName, opts)
|
|
} else {
|
|
r1 = ret.Error(1)
|
|
}
|
|
|
|
return r0, r1
|
|
}
|
|
|
|
// newS3CopyClient creates a new instance of s3CopyClient. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations.
|
|
// The first argument is typically a *testing.T value.
|
|
func newS3CopyClient(t interface {
|
|
mock.TestingT
|
|
Cleanup(func())
|
|
}) *s3CopyClient {
|
|
mock := &s3CopyClient{}
|
|
mock.Mock.Test(t)
|
|
|
|
t.Cleanup(func() { mock.AssertExpectations(t) })
|
|
|
|
return mock
|
|
}
|