mirror of
https://github.com/hanzoai/tasks.git
synced 2026-10-04 02:04:03 +00:00
Alex ShtinandGitHub
8e8f1b0f7c
## What changed?
<!-- Describe what has changed in this PR -->
Remove license header from every file. Because it is really hard to
follow in this PR here is the summary:
1. License header is removed from all `.go` and `.proto` files
:fireworks::fireworks:🎆.
2. `LICENSE` file in the root directory has only Temporal and Uber
copyrights.
3. 5 other `LICENSE` files added to the packages which have copyrights
different from Temporal and Uber: Datadog, Xargin, "Mat Ryer, Tyler
Bunnell and contributors".
4. `license_file` flag is removed from all code generation tools.
5. `copyright_file` flag is removed from `go:generate mockgen`
directive.
6. All copyright related targets are removed from `Makefile`.
7. Updated Temporal copyright year to 2025 everywhere.
## Why?
<!-- Tell your future self why have you made these changes -->
I double checked with legal department that it is not needed to have
license header in every file. One file per repo is enough. I put all
copyrights to the root `LICENSE` file and removed header from all other
files. Also updated tools and `Makefile`.
306 lines
10 KiB
Go
306 lines
10 KiB
Go
package connector_test
|
|
|
|
import (
|
|
"context"
|
|
"encoding/json"
|
|
"errors"
|
|
"io"
|
|
"os"
|
|
"strings"
|
|
"testing"
|
|
|
|
"cloud.google.com/go/storage"
|
|
"github.com/stretchr/testify/require"
|
|
"github.com/stretchr/testify/suite"
|
|
"go.temporal.io/server/common/archiver"
|
|
"go.temporal.io/server/common/archiver/gcloud/connector"
|
|
"go.temporal.io/server/common/config"
|
|
"go.uber.org/mock/gomock"
|
|
"google.golang.org/api/iterator"
|
|
)
|
|
|
|
func (s *clientSuite) SetupTest() {
|
|
s.Assertions = require.New(s.T())
|
|
s.controller = gomock.NewController(s.T())
|
|
file, _ := json.MarshalIndent(&fakeData{data: "example"}, "", " ")
|
|
|
|
path := "/tmp/temporal_archival/development"
|
|
if err := os.MkdirAll(path, os.ModePerm); err != nil {
|
|
s.FailNowf("Failed to create directory %s: %v", path, err)
|
|
}
|
|
s.Require().NoError(os.WriteFile("/tmp/temporal_archival/development/myfile.history", file, 0644))
|
|
}
|
|
|
|
func (s *clientSuite) TearDownTest() {
|
|
s.controller.Finish()
|
|
name := "/tmp/temporal_archival/development/myfile.history"
|
|
if err := os.Remove(name); err != nil {
|
|
s.Failf("Failed to remove file %s: %v", name, err)
|
|
}
|
|
}
|
|
|
|
func TestClientSuite(t *testing.T) {
|
|
suite.Run(t, new(clientSuite))
|
|
}
|
|
|
|
type clientSuite struct {
|
|
*require.Assertions
|
|
suite.Suite
|
|
controller *gomock.Controller
|
|
}
|
|
|
|
type fakeData struct {
|
|
data string
|
|
}
|
|
|
|
func (s *clientSuite) TestUpload() {
|
|
ctx := context.Background()
|
|
|
|
mockStorageClient := connector.NewMockGcloudStorageClient(s.controller)
|
|
mockBucketHandleClient := connector.NewMockBucketHandleWrapper(s.controller)
|
|
mockObjectHandler := connector.NewMockObjectHandleWrapper(s.controller)
|
|
mockWriter := connector.NewMockWriterWrapper(s.controller)
|
|
|
|
storageWrapper, _ := connector.NewClientWithParams(mockStorageClient)
|
|
|
|
mockStorageClient.EXPECT().Bucket("my-bucket-cad").Return(mockBucketHandleClient)
|
|
mockBucketHandleClient.EXPECT().Object("temporal_archival/development/myfile.history").Return(mockObjectHandler)
|
|
mockObjectHandler.EXPECT().NewWriter(ctx).Return(mockWriter)
|
|
mockWriter.EXPECT().Write(gomock.Any()).Return(2, nil)
|
|
mockWriter.EXPECT().Close().Return(nil)
|
|
|
|
URI, err := archiver.NewURI("gs://my-bucket-cad/temporal_archival/development")
|
|
s.Require().NoError(err)
|
|
err = storageWrapper.Upload(ctx, URI, "myfile.history", []byte("{}"))
|
|
s.Require().NoError(err)
|
|
}
|
|
|
|
func (s *clientSuite) TestUploadWriterCloseError() {
|
|
ctx := context.Background()
|
|
|
|
mockStorageClient := connector.NewMockGcloudStorageClient(s.controller)
|
|
mockBucketHandleClient := connector.NewMockBucketHandleWrapper(s.controller)
|
|
mockObjectHandler := connector.NewMockObjectHandleWrapper(s.controller)
|
|
mockWriter := connector.NewMockWriterWrapper(s.controller)
|
|
|
|
storageWrapper, _ := connector.NewClientWithParams(mockStorageClient)
|
|
|
|
mockStorageClient.EXPECT().Bucket("my-bucket-cad").Return(mockBucketHandleClient)
|
|
mockBucketHandleClient.EXPECT().Object("temporal_archival/development/myfile.history").Return(mockObjectHandler)
|
|
mockObjectHandler.EXPECT().NewWriter(ctx).Return(mockWriter)
|
|
mockWriter.EXPECT().Write(gomock.Any()).Return(2, nil)
|
|
mockWriter.EXPECT().Close().Return(errors.New("Not Found"))
|
|
|
|
URI, err := archiver.NewURI("gs://my-bucket-cad/temporal_archival/development")
|
|
s.Require().NoError(err)
|
|
err = storageWrapper.Upload(ctx, URI, "myfile.history", []byte("{}"))
|
|
s.Require().EqualError(err, "Not Found")
|
|
}
|
|
|
|
func (s *clientSuite) TestExist() {
|
|
ctx := context.Background()
|
|
testCases := []struct {
|
|
callContext context.Context
|
|
URI string
|
|
fileName string
|
|
bucketName string
|
|
bucketExists bool
|
|
objectExists bool
|
|
bucketExpectedError error
|
|
objectExpectedError error
|
|
}{
|
|
{
|
|
callContext: ctx,
|
|
URI: "gs://my-bucket-cad/temporal_archival/development",
|
|
fileName: "",
|
|
bucketName: "my-bucket-cad",
|
|
bucketExists: true,
|
|
objectExists: false,
|
|
bucketExpectedError: nil,
|
|
objectExpectedError: nil,
|
|
},
|
|
{
|
|
callContext: ctx,
|
|
URI: "gs://my-bucket-cad/temporal_archival/development",
|
|
fileName: "",
|
|
bucketName: "my-bucket-cad",
|
|
bucketExists: false,
|
|
objectExists: false,
|
|
bucketExpectedError: errors.New("bucket not found"),
|
|
objectExpectedError: nil,
|
|
},
|
|
{
|
|
callContext: ctx,
|
|
URI: "gs://my-bucket-cad/temporal_archival/development",
|
|
fileName: "myfile.history",
|
|
bucketName: "my-bucket-cad",
|
|
bucketExists: true,
|
|
objectExists: true,
|
|
bucketExpectedError: nil,
|
|
objectExpectedError: nil,
|
|
},
|
|
{
|
|
callContext: ctx,
|
|
URI: "gs://my-bucket-cad/temporal_archival/development",
|
|
fileName: "myfile.history",
|
|
bucketName: "my-bucket-cad",
|
|
bucketExists: true,
|
|
objectExists: false,
|
|
bucketExpectedError: nil,
|
|
objectExpectedError: errors.New("object not found"),
|
|
},
|
|
}
|
|
|
|
for _, tc := range testCases {
|
|
|
|
mockStorageClient := connector.NewMockGcloudStorageClient(s.controller)
|
|
mockBucketHandleClient := connector.NewMockBucketHandleWrapper(s.controller)
|
|
mockObjectHandler := connector.NewMockObjectHandleWrapper(s.controller)
|
|
|
|
mockStorageClient.EXPECT().Bucket(tc.bucketName).Return(mockBucketHandleClient).MaxTimes(1)
|
|
mockBucketHandleClient.EXPECT().Attrs(tc.callContext).Return(nil, tc.bucketExpectedError).MaxTimes(1)
|
|
mockBucketHandleClient.EXPECT().Object(tc.fileName).Return(mockObjectHandler).MaxTimes(1)
|
|
mockObjectHandler.EXPECT().Attrs(tc.callContext).Return(nil, tc.objectExpectedError).MaxTimes(1)
|
|
URI, _ := archiver.NewURI(tc.URI)
|
|
storageWrapper, _ := connector.NewClientWithParams(mockStorageClient)
|
|
exists, err := storageWrapper.Exist(tc.callContext, URI, tc.fileName)
|
|
if tc.bucketExists && tc.fileName == "" {
|
|
s.Require().NoError(err)
|
|
s.True(exists)
|
|
} else if !tc.bucketExists {
|
|
s.Require().Error(err)
|
|
s.Require().EqualError(err, "bucket not found")
|
|
s.False(exists)
|
|
} else if tc.bucketExists && tc.fileName != "" && tc.objectExists {
|
|
s.Require().NoError(err)
|
|
s.True(exists)
|
|
} else if tc.bucketExists && tc.fileName != "" && !tc.objectExists {
|
|
s.Require().Error(err)
|
|
s.Require().EqualError(err, "object not found")
|
|
s.False(exists)
|
|
}
|
|
}
|
|
|
|
}
|
|
|
|
func (s *clientSuite) TestGet() {
|
|
ctx := context.Background()
|
|
mockStorageClient := connector.NewMockGcloudStorageClient(s.controller)
|
|
mockBucketHandleClient := connector.NewMockBucketHandleWrapper(s.controller)
|
|
mockObjectHandler := connector.NewMockObjectHandleWrapper(s.controller)
|
|
mockReader := connector.NewMockReaderWrapper(s.controller)
|
|
|
|
storageWrapper, _ := connector.NewClientWithParams(mockStorageClient)
|
|
|
|
mockStorageClient.EXPECT().Bucket("my-bucket-cad").Return(mockBucketHandleClient)
|
|
mockBucketHandleClient.EXPECT().Object("temporal_archival/development/myfile.history").Return(mockObjectHandler)
|
|
mockObjectHandler.EXPECT().NewReader(ctx).Return(mockReader, nil)
|
|
mockReader.EXPECT().Read(gomock.Any()).Return(2, io.EOF)
|
|
mockReader.EXPECT().Close().Return(nil)
|
|
|
|
URI, err := archiver.NewURI("gs://my-bucket-cad/temporal_archival/development")
|
|
s.Require().NoError(err)
|
|
_, err = storageWrapper.Get(ctx, URI, "myfile.history")
|
|
s.Require().NoError(err)
|
|
}
|
|
|
|
func (s *clientSuite) TestWrongGoogleCredentialsPath() {
|
|
ctx := context.Background()
|
|
s.T().Setenv("GOOGLE_APPLICATION_CREDENTIALS", "/Wrong/path")
|
|
_, err := connector.NewClient(ctx, &config.GstorageArchiver{})
|
|
s.Require().Error(err)
|
|
}
|
|
|
|
func (s *clientSuite) TestQuery() {
|
|
|
|
ctx := context.Background()
|
|
mockBucketHandleClient := connector.NewMockBucketHandleWrapper(s.controller)
|
|
mockStorageClient := connector.NewMockGcloudStorageClient(s.controller)
|
|
mockObjectIterator := connector.NewMockObjectIteratorWrapper(s.controller)
|
|
storageWrapper, _ := connector.NewClientWithParams(mockStorageClient)
|
|
|
|
attr := new(storage.ObjectAttrs)
|
|
attr.Name = "fileName_01"
|
|
|
|
mockStorageClient.EXPECT().Bucket("my-bucket-cad").Return(mockBucketHandleClient)
|
|
mockBucketHandleClient.EXPECT().Objects(ctx, gomock.Any()).Return(mockObjectIterator)
|
|
mockIterator := 0
|
|
mockObjectIterator.EXPECT().Next().DoAndReturn(func() (*storage.ObjectAttrs, error) {
|
|
mockIterator++
|
|
if mockIterator == 1 {
|
|
return attr, nil
|
|
}
|
|
return nil, iterator.Done
|
|
|
|
}).Times(2)
|
|
|
|
var fileNames []string
|
|
URI, err := archiver.NewURI("gs://my-bucket-cad/temporal_archival/development")
|
|
s.Require().NoError(err)
|
|
fileNames, err = storageWrapper.Query(ctx, URI, "7478875943689868082123907395549832634615673687049942026838")
|
|
s.Require().NoError(err)
|
|
s.Equal(strings.Join(fileNames, ", "), "fileName_01")
|
|
}
|
|
|
|
func (s *clientSuite) TestQueryWithFilter() {
|
|
|
|
ctx := context.Background()
|
|
mockBucketHandleClient := connector.NewMockBucketHandleWrapper(s.controller)
|
|
mockStorageClient := connector.NewMockGcloudStorageClient(s.controller)
|
|
mockObjectIterator := connector.NewMockObjectIteratorWrapper(s.controller)
|
|
storageWrapper, _ := connector.NewClientWithParams(mockStorageClient)
|
|
|
|
attr := new(storage.ObjectAttrs)
|
|
attr.Name = "closeTimeout_2020-02-27T09:42:28Z_12851121011173788097_4418294404690464320_15619178330501475177.visibility"
|
|
attrInvalid := new(storage.ObjectAttrs)
|
|
attrInvalid.Name = "closeTimeout_2020-02-27T09:42:28Z_12851121011173788097_4418294404690464321_15619178330501475177.visibility"
|
|
|
|
mockStorageClient.EXPECT().Bucket("my-bucket-cad").Return(mockBucketHandleClient)
|
|
mockBucketHandleClient.EXPECT().Objects(ctx, gomock.Any()).Return(mockObjectIterator)
|
|
mockIterator := 0
|
|
mockObjectIterator.EXPECT().Next().DoAndReturn(func() (*storage.ObjectAttrs, error) {
|
|
mockIterator++
|
|
switch mockIterator {
|
|
case 1:
|
|
return attr, nil
|
|
case 2:
|
|
return attrInvalid, nil
|
|
default:
|
|
return nil, iterator.Done
|
|
}
|
|
|
|
}).Times(3)
|
|
|
|
var fileNames []string
|
|
URI, err := archiver.NewURI("gs://my-bucket-cad/temporal_archival/development")
|
|
s.Require().NoError(err)
|
|
fileNames, _, _, err = storageWrapper.QueryWithFilters(ctx, URI, "closeTimeout_2020-02-27T09:42:28Z", 0, 0, []connector.Precondition{newWorkflowIDPrecondition("4418294404690464320")})
|
|
|
|
s.Require().NoError(err)
|
|
s.Equal(strings.Join(fileNames, ", "), "closeTimeout_2020-02-27T09:42:28Z_12851121011173788097_4418294404690464320_15619178330501475177.visibility")
|
|
}
|
|
|
|
func newWorkflowIDPrecondition(workflowID string) connector.Precondition {
|
|
return func(subject interface{}) bool {
|
|
|
|
if workflowID == "" {
|
|
return true
|
|
}
|
|
|
|
fileName, ok := subject.(string)
|
|
if !ok {
|
|
return false
|
|
}
|
|
|
|
if strings.Contains(fileName, workflowID) {
|
|
fileNameParts := strings.Split(fileName, "_")
|
|
if len(fileNameParts) != 5 {
|
|
return true
|
|
}
|
|
return strings.Contains(fileName, fileNameParts[3])
|
|
}
|
|
|
|
return false
|
|
}
|
|
}
|