mirror of
https://github.com/hanzoai/team.git
synced 2026-09-30 01:57:35 +00:00
The server module named github.com/hanzoai/team/server/v8 — a path that resolves to the retired workspace's repository, not this one — while server/public still published github.com/mattermost/mattermost/server/public. Both are github.com/hanzoteam/server/... now, which is where this code actually lives. That second half is what made the binary unbootable. Upstream's public path stayed claimed by upstream, mattermost-plugin-ai imported it, and the build linked public/model twice; public/plugin's gob.Register then ran twice under one name and panicked at init, before main. Reverting the rename traded the brand for a working binary. Rebranding the plugin alongside it needs no such trade: the agents fork is github.com/hanzoteam/agents/v2 against our public module, so no module named mattermost/mattermost is in the graph at all. Measured: `go list -deps ./cmd/mattermost` names exactly one public/model, and it is ours.
830 lines
29 KiB
Go
830 lines
29 KiB
Go
// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
|
|
// See LICENSE.txt for license information.
|
|
|
|
package filestore
|
|
|
|
import (
|
|
"archive/zip"
|
|
"bytes"
|
|
"context"
|
|
"crypto/tls"
|
|
"encoding/base64"
|
|
"errors"
|
|
"fmt"
|
|
"io"
|
|
"net/http"
|
|
"net/url"
|
|
"path"
|
|
"strings"
|
|
"time"
|
|
|
|
"github.com/Azure/azure-sdk-for-go/sdk/azcore"
|
|
"github.com/Azure/azure-sdk-for-go/sdk/azidentity"
|
|
"github.com/Azure/azure-sdk-for-go/sdk/storage/azblob"
|
|
"github.com/Azure/azure-sdk-for-go/sdk/storage/azblob/blob"
|
|
"github.com/Azure/azure-sdk-for-go/sdk/storage/azblob/bloberror"
|
|
"github.com/Azure/azure-sdk-for-go/sdk/storage/azblob/blockblob"
|
|
"github.com/Azure/azure-sdk-for-go/sdk/storage/azblob/container"
|
|
"github.com/Azure/azure-sdk-for-go/sdk/storage/azblob/sas"
|
|
"github.com/Azure/azure-sdk-for-go/sdk/storage/azblob/service"
|
|
"github.com/google/uuid"
|
|
"github.com/hanzoteam/server/server/public/model"
|
|
"github.com/hanzoteam/server/server/public/shared/httpservice"
|
|
"github.com/hanzoteam/server/server/public/shared/mlog"
|
|
)
|
|
|
|
// azureBlockSize is the chunk size used when staging block blob uploads.
|
|
// Matches the Azure SDK's default block size for UploadStream and keeps each
|
|
// StageBlock call well under the per-block REST limit (4000 MiB).
|
|
const azureBlockSize = 4 * 1024 * 1024
|
|
|
|
// AzureFileBackend stores files in Azure Blob Storage. Two authentication
|
|
// modes are supported: shared key (an account access key configured by the
|
|
// admin) and Microsoft Entra ID via DefaultAzureCredential (managed identity,
|
|
// service principal, workload identity, or az login - whichever the host
|
|
// environment provides).
|
|
type AzureFileBackend struct {
|
|
client *azblob.Client
|
|
// sharedKey holds the account credential when authMode is shared key, and is
|
|
// nil under default credential. GeneratePublicLink needs it to sign a Service
|
|
// SAS by hand, since GetSASURL does not allow to change the ContentDisposition
|
|
// if not at the blob level, and the parsing it does of the URL to retrieve
|
|
// container and blob names does not support custom endpoints.
|
|
sharedKey *azblob.SharedKeyCredential
|
|
authMode string
|
|
container string
|
|
pathPrefix string
|
|
timeout time.Duration
|
|
presignExpires time.Duration
|
|
ssl bool
|
|
}
|
|
|
|
var _ FileBackendWithLinkGenerator = (*AzureFileBackend)(nil)
|
|
|
|
func NewAzureFileBackend(settings FileBackendSettings) (*AzureFileBackend, error) {
|
|
if err := settings.CheckMandatoryAzureFields(); err != nil {
|
|
return nil, err
|
|
}
|
|
|
|
scheme := "https"
|
|
if !settings.AzureSSL {
|
|
scheme = "http"
|
|
}
|
|
|
|
serviceURL, err := buildAzureServiceURL(settings.AzureCloud, scheme, settings.AzureStorageAccount, settings.AzureEndpoint)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
|
|
client, sharedKey, err := newAzureClient(settings, serviceURL, buildAzureClientOptions(settings))
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
|
|
// Config.IsValid rejects non-positive timeouts before they reach this
|
|
// constructor, but direct callers (tests, library users that build a
|
|
// FileBackendSettings by hand) can still slip a zero or negative value
|
|
// in. Fall back to a sane default in that case, and log loudly enough
|
|
// for the substitution to show up if it ever happens in production.
|
|
timeout := time.Duration(settings.AzureRequestTimeoutMilliseconds) * time.Millisecond
|
|
if timeout <= 0 {
|
|
mlog.Warn("AzureRequestTimeoutMilliseconds is non-positive; falling back to 30s default",
|
|
mlog.Int("value", int(settings.AzureRequestTimeoutMilliseconds)))
|
|
timeout = 30 * time.Second
|
|
}
|
|
|
|
return &AzureFileBackend{
|
|
client: client,
|
|
sharedKey: sharedKey,
|
|
authMode: settings.AzureAuthMode,
|
|
container: settings.AzureContainer,
|
|
pathPrefix: settings.AzurePathPrefix,
|
|
timeout: timeout,
|
|
presignExpires: time.Duration(settings.AzurePresignExpiresSeconds) * time.Second,
|
|
ssl: settings.AzureSSL,
|
|
}, nil
|
|
}
|
|
|
|
func (b *AzureFileBackend) DriverName() string {
|
|
return driverAzure
|
|
}
|
|
|
|
// newAzureClient builds an azblob client for the configured authentication
|
|
// mode. Shared key uses NewClientWithSharedKeyCredential; default credential
|
|
// uses NewClient with DefaultAzureCredential, which discovers managed
|
|
// identity, workload identity, service principal env vars, and az login in
|
|
// that order at runtime.
|
|
//
|
|
// The shared-key credential is returned alongside the client when shared-key
|
|
// auth is in use, so callers (GeneratePublicLink in particular) can sign
|
|
// Service SAS tokens without round-tripping to Entra ID. It is nil in the
|
|
// default-credential path; that path obtains a user-delegation credential
|
|
// lazily at link-generation time.
|
|
func newAzureClient(settings FileBackendSettings, serviceURL string, clientOptions *azblob.ClientOptions) (*azblob.Client, *azblob.SharedKeyCredential, error) {
|
|
switch settings.AzureAuthMode {
|
|
case model.AzureAuthModeDefaultCredential:
|
|
cred, err := azidentity.NewDefaultAzureCredential(nil)
|
|
if err != nil {
|
|
return nil, nil, fmt.Errorf("failed to create azure default credential: %w", err)
|
|
}
|
|
client, err := azblob.NewClient(serviceURL, cred, clientOptions)
|
|
if err != nil {
|
|
return nil, nil, fmt.Errorf("failed to create azure blob client: %w", err)
|
|
}
|
|
return client, nil, nil
|
|
case model.AzureAuthModeSharedKey:
|
|
cred, err := azblob.NewSharedKeyCredential(settings.AzureStorageAccount, settings.AzureAccessKey)
|
|
if err != nil {
|
|
return nil, nil, fmt.Errorf("failed to create azure shared key credential: %w", err)
|
|
}
|
|
client, err := azblob.NewClientWithSharedKeyCredential(serviceURL, cred, clientOptions)
|
|
if err != nil {
|
|
return nil, nil, fmt.Errorf("failed to create azure blob client: %w", err)
|
|
}
|
|
return client, cred, nil
|
|
default:
|
|
return nil, nil, fmt.Errorf("unknown azure auth mode %q", settings.AzureAuthMode)
|
|
}
|
|
}
|
|
|
|
// buildAzureClientOptions selects the HTTP transport for the Azure SDK. The
|
|
// custom cloud routes requests through httpservice's transport, honoring
|
|
// AllowedUntrustedInternalConnections. The commercial and government clouds use
|
|
// a fixed Azure host (which may resolve to a private address under Private
|
|
// Link), so they keep the SDK default transport and only override it to honor
|
|
// SkipVerify, mirroring the S3 backend.
|
|
func buildAzureClientOptions(settings FileBackendSettings) *azblob.ClientOptions {
|
|
if settings.AzureCloud == model.AzureCloudCustom {
|
|
return &azblob.ClientOptions{
|
|
ClientOptions: azcore.ClientOptions{
|
|
Transport: &http.Client{
|
|
Transport: httpservice.NewTransportForInternalConnections(settings.SkipVerify, settings.AllowedUntrustedInternalConnections),
|
|
},
|
|
},
|
|
}
|
|
}
|
|
|
|
if settings.SkipVerify {
|
|
return &azblob.ClientOptions{
|
|
ClientOptions: azcore.ClientOptions{
|
|
Transport: &http.Client{
|
|
Transport: &http.Transport{
|
|
TLSClientConfig: &tls.Config{InsecureSkipVerify: true},
|
|
},
|
|
},
|
|
},
|
|
}
|
|
}
|
|
|
|
return nil
|
|
}
|
|
|
|
// buildAzureServiceURL renders the Blob service URL that the SDK signs
|
|
// requests against. The cloud value selects the topology:
|
|
//
|
|
// - commercial -> vhost-style against blob.core.windows.net, e.g.
|
|
// https://{account}.blob.core.windows.net/.
|
|
// - government -> vhost-style against blob.core.usgovcloudapi.net,
|
|
// e.g. https://{account}.blob.core.usgovcloudapi.net/.
|
|
// - custom -> the admin-provided endpoint is the full service URL,
|
|
// including scheme and storage account (vhost-style for production
|
|
// Azure, path-style for Azurite or reverse proxies). Hanzo Team
|
|
// does not modify the URL.
|
|
//
|
|
// Empty cloud is treated as commercial so existing configs that pre-date
|
|
// this field keep working. Shared-key auth signs against the URL host,
|
|
// so for custom deployments the admin is responsible for ensuring the
|
|
// host actually serves the storage account named in the credential.
|
|
func buildAzureServiceURL(cloud, scheme, account, endpoint string) (string, error) {
|
|
switch cloud {
|
|
case model.AzureCloudCommercial, "":
|
|
if !model.IsValidAzureStorageAccountName(account) {
|
|
return "", fmt.Errorf("invalid azure storage account name %q", account)
|
|
}
|
|
return fmt.Sprintf("%s://%s.blob.core.windows.net/", scheme, account), nil
|
|
case model.AzureCloudGovernment:
|
|
if !model.IsValidAzureStorageAccountName(account) {
|
|
return "", fmt.Errorf("invalid azure storage account name %q", account)
|
|
}
|
|
return fmt.Sprintf("%s://%s.blob.core.usgovcloudapi.net/", scheme, account), nil
|
|
case model.AzureCloudCustom:
|
|
if endpoint == "" {
|
|
return "", errors.New("AzureCloud=custom requires AzureEndpoint to be set")
|
|
}
|
|
// The admin owns this URL end to end, but we still reject inputs
|
|
// that the SDK is guaranteed to fail on later (missing scheme,
|
|
// missing host, a scheme other than http/https) so the
|
|
// failure mode is a clear configuration error at startup rather
|
|
// than an opaque SDK error on the first blob request.
|
|
parsed, err := url.Parse(endpoint)
|
|
if err != nil {
|
|
return "", fmt.Errorf("AzureEndpoint is not a valid URL: %w", err)
|
|
}
|
|
if parsed.Scheme != "http" && parsed.Scheme != "https" {
|
|
return "", fmt.Errorf("AzureEndpoint must use http or https, got %q", endpoint)
|
|
}
|
|
if parsed.Host == "" {
|
|
return "", fmt.Errorf("AzureEndpoint must include a host, got %q", endpoint)
|
|
}
|
|
return endpoint, nil
|
|
default:
|
|
return "", fmt.Errorf("unknown AzureCloud value %q", cloud)
|
|
}
|
|
}
|
|
|
|
// prefix joins the configured pathPrefix and the caller-supplied path.
|
|
// Using a plain path.Join, a value like "foo/../../secret" can escape
|
|
// the prefix entirely, so we compute the join and verify the result is
|
|
// the prefix directory itself or a descendant of it. The descendant check
|
|
// requires a path-separator boundary so a prefix of "mattermost" does not
|
|
// match a sibling like "mattermost-evil/...". If the joined path escapes,
|
|
// we fall back to joining the prefix with path.Base, which may drop any
|
|
// intermediate directories the caller intended.
|
|
func (b *AzureFileBackend) prefix(p string) string {
|
|
joined := path.Join(b.pathPrefix, p)
|
|
if b.pathPrefix == "" {
|
|
return joined
|
|
}
|
|
|
|
cleanPrefix := strings.TrimSuffix(path.Clean(b.pathPrefix), "/")
|
|
if joined == cleanPrefix || strings.HasPrefix(joined, cleanPrefix+"/") {
|
|
return joined
|
|
}
|
|
return path.Join(cleanPrefix, path.Base(p))
|
|
}
|
|
|
|
func (b *AzureFileBackend) newBlobClient(p string) *blob.Client {
|
|
return b.client.ServiceClient().NewContainerClient(b.container).NewBlobClient(b.prefix(p))
|
|
}
|
|
|
|
func (b *AzureFileBackend) newBlockBlobClient(p string) *blockblob.Client {
|
|
return b.client.ServiceClient().NewContainerClient(b.container).NewBlockBlobClient(b.prefix(p))
|
|
}
|
|
|
|
func (b *AzureFileBackend) newContainerClient() *container.Client {
|
|
return b.client.ServiceClient().NewContainerClient(b.container)
|
|
}
|
|
|
|
// TestConnection probes the configured container and reports the outcome
|
|
// using the typed errors shared with the other backends. Container
|
|
// creation is deliberately out of scope here - callers (Server.Start)
|
|
// decide whether to provision a missing container via MakeContainer.
|
|
// That separation keeps a typo in the System Console from silently
|
|
// provisioning an unwanted container, and matches the S3 contract where
|
|
// TestConnection returns FileBackendNoBucketError and MakeBucket is an
|
|
// explicit call.
|
|
func (b *AzureFileBackend) TestConnection() error {
|
|
ctx, cancel := context.WithTimeout(context.Background(), b.timeout)
|
|
defer cancel()
|
|
|
|
_, err := b.newContainerClient().GetProperties(ctx, nil)
|
|
if err == nil {
|
|
return nil
|
|
}
|
|
if bloberror.HasCode(err, bloberror.ContainerNotFound) {
|
|
return &FileBackendNoBucketError{Err: fmt.Errorf("azure container %q does not exist: %w", b.container, err)}
|
|
}
|
|
if isAzureAuthError(err) {
|
|
return &FileBackendAuthError{Err: fmt.Errorf("unable to authenticate against azure blob storage: %w", err)}
|
|
}
|
|
return fmt.Errorf("unable to connect to azure blob storage: %w", err)
|
|
}
|
|
|
|
// MakeContainer creates the configured container. Mirrors S3FileBackend.MakeBucket
|
|
// so callers can opt into container provisioning explicitly. An already-existing
|
|
// container is treated as success so that concurrent boots (two nodes racing
|
|
// through TestConnection plus MakeContainer) both converge cleanly.
|
|
func (b *AzureFileBackend) MakeContainer() error {
|
|
ctx, cancel := context.WithTimeout(context.Background(), b.timeout)
|
|
defer cancel()
|
|
|
|
if _, err := b.newContainerClient().Create(ctx, nil); err != nil {
|
|
if bloberror.HasCode(err, bloberror.ContainerAlreadyExists) {
|
|
return nil
|
|
}
|
|
return fmt.Errorf("unable to create azure container %q: %w", b.container, err)
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func (b *AzureFileBackend) Reader(p string) (ReadCloseSeeker, error) {
|
|
// Arm the deadline *before* the first network call, then hand the same
|
|
// timer to the returned reader on success. The previous code only set up
|
|
// the timer on the happy path, which left GetProperties running against a
|
|
// no-deadline context.
|
|
ctx, cancel := context.WithCancel(context.Background())
|
|
timer := time.AfterFunc(b.timeout, cancel)
|
|
blobClient := b.newBlobClient(p)
|
|
|
|
props, err := blobClient.GetProperties(ctx, nil)
|
|
if err != nil {
|
|
timer.Stop()
|
|
cancel()
|
|
return nil, fmt.Errorf("unable to read file %q: %w", p, err)
|
|
}
|
|
if props.ContentLength == nil {
|
|
timer.Stop()
|
|
cancel()
|
|
return nil, fmt.Errorf("missing content length for %q", p)
|
|
}
|
|
|
|
return &azureRangeReader{
|
|
ctx: ctx,
|
|
cancel: cancel,
|
|
timer: timer,
|
|
blobClient: blobClient,
|
|
size: *props.ContentLength,
|
|
}, nil
|
|
}
|
|
|
|
func (b *AzureFileBackend) ReadFile(p string) ([]byte, error) {
|
|
r, err := b.Reader(p)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
defer r.Close()
|
|
return io.ReadAll(r)
|
|
}
|
|
|
|
func (b *AzureFileBackend) FileExists(p string) (bool, error) {
|
|
ctx, cancel := context.WithTimeout(context.Background(), b.timeout)
|
|
defer cancel()
|
|
|
|
_, err := b.newBlobClient(p).GetProperties(ctx, nil)
|
|
if err != nil {
|
|
if bloberror.HasCode(err, bloberror.BlobNotFound) {
|
|
return false, nil
|
|
}
|
|
return false, fmt.Errorf("unable to check existence of %q: %w", p, err)
|
|
}
|
|
return true, nil
|
|
}
|
|
|
|
func (b *AzureFileBackend) FileSize(p string) (int64, error) {
|
|
ctx, cancel := context.WithTimeout(context.Background(), b.timeout)
|
|
defer cancel()
|
|
|
|
props, err := b.newBlobClient(p).GetProperties(ctx, nil)
|
|
if err != nil {
|
|
return 0, fmt.Errorf("unable to get size of %q: %w", p, err)
|
|
}
|
|
|
|
return model.SafeDereference(props.ContentLength), nil
|
|
}
|
|
|
|
func (b *AzureFileBackend) FileModTime(p string) (time.Time, error) {
|
|
ctx, cancel := context.WithTimeout(context.Background(), b.timeout)
|
|
defer cancel()
|
|
|
|
props, err := b.newBlobClient(p).GetProperties(ctx, nil)
|
|
if err != nil {
|
|
return time.Time{}, fmt.Errorf("unable to get modification time of %q: %w", p, err)
|
|
}
|
|
|
|
return model.SafeDereference(props.LastModified), nil
|
|
}
|
|
|
|
// CopyFile copies via StartCopyFromURL and polls the resulting blob's copy
|
|
// status until it succeeds, matching the synchronous semantics that the
|
|
// FileBackend interface (and the S3 driver via ComposeObject) provides.
|
|
func (b *AzureFileBackend) CopyFile(oldPath, newPath string) error {
|
|
ctx, cancel := context.WithTimeout(context.Background(), b.timeout)
|
|
defer cancel()
|
|
|
|
src := b.newBlobClient(oldPath).URL()
|
|
dst := b.newBlockBlobClient(newPath)
|
|
if _, err := dst.StartCopyFromURL(ctx, src, nil); err != nil {
|
|
return fmt.Errorf("unable to copy %q to %q: %w", oldPath, newPath, err)
|
|
}
|
|
|
|
// Poll until the copy reports success. For server-to-server copies within
|
|
// the same account this is typically synchronous, but the API is
|
|
// asynchronous in general, so we wait.
|
|
for {
|
|
props, err := dst.GetProperties(ctx, nil)
|
|
if err != nil {
|
|
return fmt.Errorf("unable to read copy status for %q: %w", newPath, err)
|
|
}
|
|
if props.CopyStatus == nil {
|
|
return nil
|
|
}
|
|
switch *props.CopyStatus {
|
|
case blob.CopyStatusTypeSuccess:
|
|
return nil
|
|
case blob.CopyStatusTypeFailed, blob.CopyStatusTypeAborted:
|
|
desc := model.SafeDereference(props.CopyStatusDescription)
|
|
return fmt.Errorf("azure copy from %q to %q ended in status %q: %q", oldPath, newPath, *props.CopyStatus, desc)
|
|
}
|
|
select {
|
|
case <-ctx.Done():
|
|
return fmt.Errorf("azure copy from %q to %q did not complete in time: %w", oldPath, newPath, ctx.Err())
|
|
case <-time.After(50 * time.Millisecond):
|
|
}
|
|
}
|
|
}
|
|
|
|
func (b *AzureFileBackend) MoveFile(oldPath, newPath string) error {
|
|
if err := b.CopyFile(oldPath, newPath); err != nil {
|
|
return err
|
|
}
|
|
return b.RemoveFile(oldPath)
|
|
}
|
|
|
|
func (b *AzureFileBackend) WriteFile(fr io.Reader, p string) (int64, error) {
|
|
ctx, cancel := context.WithTimeout(context.Background(), b.timeout)
|
|
defer cancel()
|
|
return b.WriteFileContext(ctx, fr, p)
|
|
}
|
|
|
|
// stageBlocks reads fr in azureBlockSize chunks and stages each chunk as a
|
|
// block under a fresh ID. Returns the IDs of the newly staged blocks (in
|
|
// order) and the total byte count. The caller is responsible for committing
|
|
// the block list.
|
|
func (b *AzureFileBackend) stageBlocks(ctx context.Context, bb *blockblob.Client, fr io.Reader, p string) ([]string, int64, error) {
|
|
buf := make([]byte, azureBlockSize)
|
|
var ids []string
|
|
var total int64
|
|
|
|
for {
|
|
n, err := io.ReadFull(fr, buf)
|
|
if n > 0 {
|
|
id, idErr := newAzureBlockID()
|
|
if idErr != nil {
|
|
return nil, 0, fmt.Errorf("failed to generate azure block id: %w", idErr)
|
|
}
|
|
if _, sbErr := bb.StageBlock(ctx, id, &readSeekNopCloser{Reader: bytes.NewReader(buf[:n])}, nil); sbErr != nil {
|
|
return nil, 0, fmt.Errorf("unable to stage block for %q: %w", p, sbErr)
|
|
}
|
|
ids = append(ids, id)
|
|
total += int64(n)
|
|
}
|
|
if err == io.EOF || err == io.ErrUnexpectedEOF {
|
|
break
|
|
}
|
|
if err != nil {
|
|
return nil, 0, fmt.Errorf("failed to read input: %w", err)
|
|
}
|
|
}
|
|
return ids, total, nil
|
|
}
|
|
|
|
// WriteFileContext stages the body in fixed-size blocks and commits a fresh
|
|
// block list. It deliberately does not use the SDK's UploadStream helper:
|
|
// UploadStream's small-payload fast path falls back to single-shot PutBlob,
|
|
// which leaves the resulting blob with no committed block list. A subsequent
|
|
// AppendFile that calls CommitBlockList on that blob would then clobber its
|
|
// content. Routing every WriteFile through StageBlock + CommitBlockList keeps
|
|
// AppendFile correct regardless of payload size.
|
|
//
|
|
// The caller's context governs the entire upload - no inner timeout is added.
|
|
// TryWriteFileContext (filesstore.go) relies on this to let long-running
|
|
// callers like message-export bulk writes opt out of the per-operation
|
|
// timeout that WriteFile applies by default.
|
|
func (b *AzureFileBackend) WriteFileContext(ctx context.Context, fr io.Reader, p string) (int64, error) {
|
|
bb := b.newBlockBlobClient(p)
|
|
blockIDs, total, err := b.stageBlocks(ctx, bb, fr, p)
|
|
if err != nil {
|
|
return 0, err
|
|
}
|
|
|
|
if len(blockIDs) == 0 {
|
|
// Empty input - still need to materialize an empty blob with a
|
|
// committed block list so AppendFile can target it.
|
|
id, idErr := newAzureBlockID()
|
|
if idErr != nil {
|
|
return 0, fmt.Errorf("failed to generate azure block id: %w", idErr)
|
|
}
|
|
if _, sbErr := bb.StageBlock(ctx, id, &readSeekNopCloser{Reader: bytes.NewReader(nil)}, nil); sbErr != nil {
|
|
return 0, fmt.Errorf("unable to stage empty block for %q: %w", p, sbErr)
|
|
}
|
|
blockIDs = append(blockIDs, id)
|
|
}
|
|
|
|
if _, err := bb.CommitBlockList(ctx, blockIDs, nil); err != nil {
|
|
return 0, fmt.Errorf("unable to commit block list for %q: %w", p, err)
|
|
}
|
|
return total, nil
|
|
}
|
|
|
|
// AppendFile stages the new chunk as one or more blocks and commits the
|
|
// existing committed block list plus the newly staged IDs. Each AppendFile
|
|
// call uploads the new bytes exactly once - no re-download, no
|
|
// re-concatenate, no re-upload of the prior contents. The S3-style contract
|
|
// is preserved: returns an error if the target blob does not yet exist;
|
|
// returns the number of bytes appended (not the resulting total size).
|
|
//
|
|
// Refuses to append to a blob that has content but no committed block list
|
|
// (i.e. was uploaded via Put Blob by another tool - Azure portal, azcopy,
|
|
// a migration script). Committing a new block list against such a blob
|
|
// would replace the existing content with only the appended bytes, so
|
|
// failing loud beats silent data loss.
|
|
func (b *AzureFileBackend) AppendFile(fr io.Reader, p string) (int64, error) {
|
|
ctx, cancel := context.WithTimeout(context.Background(), b.timeout)
|
|
defer cancel()
|
|
|
|
bb := b.newBlockBlobClient(p)
|
|
|
|
listResp, err := bb.GetBlockList(ctx, blockblob.BlockListTypeCommitted, nil)
|
|
if err != nil {
|
|
return 0, fmt.Errorf("unable to find file %q to append data: %w", p, err)
|
|
}
|
|
|
|
var existingIDs []string
|
|
if listResp.BlockList.CommittedBlocks != nil {
|
|
for _, blk := range listResp.BlockList.CommittedBlocks {
|
|
if blk.Name != nil {
|
|
existingIDs = append(existingIDs, *blk.Name)
|
|
}
|
|
}
|
|
}
|
|
|
|
if len(existingIDs) == 0 {
|
|
props, propsErr := bb.GetProperties(ctx, nil)
|
|
if propsErr != nil {
|
|
return 0, fmt.Errorf("unable to inspect %q before append: %w", p, propsErr)
|
|
}
|
|
if model.SafeDereference(props.ContentLength) > 0 {
|
|
return 0, fmt.Errorf("refusing to append to %q: blob has content but no committed block list (likely written via Put Blob by another tool)", p)
|
|
}
|
|
}
|
|
|
|
newIDs, total, err := b.stageBlocks(ctx, bb, fr, p)
|
|
if err != nil {
|
|
return 0, err
|
|
}
|
|
|
|
if _, err := bb.CommitBlockList(ctx, append(existingIDs, newIDs...), nil); err != nil {
|
|
return 0, fmt.Errorf("unable to commit block list for %q: %w", p, err)
|
|
}
|
|
return total, nil
|
|
}
|
|
|
|
// GeneratePublicLink returns a time-limited, read-only URL to the blob at path
|
|
// using a Shared Access Signature. The SAS is auth-mode aware:
|
|
//
|
|
// - shared-key auth signs a Service SAS in-process with the stored
|
|
// SharedKeyCredential.
|
|
// - default-credential auth fetches a user-delegation key from Entra ID
|
|
// (one round trip per call) and signs a user-delegation SAS with it.
|
|
//
|
|
// This is intended for the export-download flow (App.GeneratePresignURLForExport
|
|
// and the /exportlink slash command). End users never reach this code path.
|
|
func (b *AzureFileBackend) GeneratePublicLink(path string) (string, time.Duration, error) {
|
|
if b.presignExpires <= 0 {
|
|
return "", 0, errors.New("azure presign expiration is not configured")
|
|
}
|
|
|
|
prefixed := b.prefix(path)
|
|
|
|
// Back-date the start by a small fixed amount to absorb minor clock skew
|
|
// between this host and Azure, matching the azure-sdk-for-go SAS examples:
|
|
// https://github.com/Azure/azure-sdk-for-go/blob/65c3b792856d9ad7ce0b59c127ce299358e41a01/sdk/storage/azblob/sas/examples_test.go#L71
|
|
// https://github.com/Azure/azure-sdk-for-go/blob/65c3b792856d9ad7ce0b59c127ce299358e41a01/sdk/storage/azblob/service/examples_test.go#L315
|
|
const clockSkew = 10 * time.Second
|
|
start := time.Now().UTC().Add(-clockSkew)
|
|
expiry := start.Add(b.presignExpires)
|
|
|
|
protocol := sas.ProtocolHTTPSandHTTP
|
|
if b.ssl {
|
|
protocol = sas.ProtocolHTTPS
|
|
}
|
|
|
|
values := sas.BlobSignatureValues{
|
|
Protocol: protocol,
|
|
StartTime: start,
|
|
ExpiryTime: expiry,
|
|
Permissions: (&sas.BlobPermissions{Read: true}).String(),
|
|
ContainerName: b.container,
|
|
BlobName: prefixed,
|
|
// Request browsers to download the file instead of rendering it
|
|
ContentDisposition: "attachment",
|
|
}
|
|
|
|
var (
|
|
qps sas.QueryParameters
|
|
err error
|
|
)
|
|
switch b.authMode {
|
|
case model.AzureAuthModeSharedKey:
|
|
if b.sharedKey == nil {
|
|
return "", 0, errors.New("shared key credential is unavailable")
|
|
}
|
|
qps, err = values.SignWithSharedKey(b.sharedKey)
|
|
if err != nil {
|
|
return "", 0, fmt.Errorf("unable to sign service SAS for %q: %w", path, err)
|
|
}
|
|
case model.AzureAuthModeDefaultCredential:
|
|
ctx, cancel := context.WithTimeout(context.Background(), b.timeout)
|
|
defer cancel()
|
|
udc, udcErr := b.client.ServiceClient().GetUserDelegationCredential(ctx, service.KeyInfo{
|
|
Start: new(start.Format(sas.TimeFormat)),
|
|
Expiry: new(expiry.Format(sas.TimeFormat)),
|
|
}, nil)
|
|
if udcErr != nil {
|
|
return "", 0, fmt.Errorf("unable to obtain user delegation key for %q: %w", path, udcErr)
|
|
}
|
|
qps, err = values.SignWithUserDelegation(udc)
|
|
if err != nil {
|
|
return "", 0, fmt.Errorf("unable to sign user-delegation SAS for %q: %w", path, err)
|
|
}
|
|
default:
|
|
return "", 0, fmt.Errorf("unknown azure auth mode %q", b.authMode)
|
|
}
|
|
|
|
return b.newBlobClient(path).URL() + "?" + qps.Encode(), b.presignExpires, nil
|
|
}
|
|
|
|
func (b *AzureFileBackend) RemoveFile(p string) error {
|
|
ctx, cancel := context.WithTimeout(context.Background(), b.timeout)
|
|
defer cancel()
|
|
|
|
_, err := b.newBlobClient(p).Delete(ctx, nil)
|
|
if err != nil && !bloberror.HasCode(err, bloberror.BlobNotFound) {
|
|
return fmt.Errorf("unable to remove file %q: %w", p, err)
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func (b *AzureFileBackend) ListDirectory(p string) ([]string, error) {
|
|
prefix := b.prefix(p)
|
|
if prefix != "" && !strings.HasSuffix(prefix, "/") {
|
|
prefix += "/"
|
|
}
|
|
|
|
ctx, cancel := context.WithTimeout(context.Background(), b.timeout)
|
|
defer cancel()
|
|
|
|
pager := b.newContainerClient().NewListBlobsHierarchyPager("/", &container.ListBlobsHierarchyOptions{
|
|
Prefix: &prefix,
|
|
})
|
|
|
|
var entries []string
|
|
for pager.More() {
|
|
page, err := pager.NextPage(ctx)
|
|
if err != nil {
|
|
return nil, fmt.Errorf("unable to list directory %q: %w", p, err)
|
|
}
|
|
for _, item := range page.Segment.BlobItems {
|
|
if item.Name == nil {
|
|
continue
|
|
}
|
|
name := strings.TrimPrefix(*item.Name, b.pathPrefix)
|
|
name = strings.TrimPrefix(name, "/")
|
|
entries = append(entries, name)
|
|
}
|
|
for _, item := range page.Segment.BlobPrefixes {
|
|
if item.Name == nil {
|
|
continue
|
|
}
|
|
name := strings.TrimPrefix(*item.Name, b.pathPrefix)
|
|
name = strings.TrimPrefix(name, "/")
|
|
name = strings.TrimSuffix(name, "/")
|
|
entries = append(entries, name)
|
|
}
|
|
}
|
|
return entries, nil
|
|
}
|
|
|
|
func (b *AzureFileBackend) ListDirectoryRecursively(p string) ([]string, error) {
|
|
prefix := b.prefix(p)
|
|
if prefix != "" && !strings.HasSuffix(prefix, "/") {
|
|
prefix += "/"
|
|
}
|
|
|
|
ctx, cancel := context.WithTimeout(context.Background(), b.timeout)
|
|
defer cancel()
|
|
|
|
pager := b.newContainerClient().NewListBlobsFlatPager(&container.ListBlobsFlatOptions{
|
|
Prefix: &prefix,
|
|
})
|
|
|
|
var entries []string
|
|
for pager.More() {
|
|
page, err := pager.NextPage(ctx)
|
|
if err != nil {
|
|
return nil, fmt.Errorf("unable to list directory %q recursively: %w", p, err)
|
|
}
|
|
for _, item := range page.Segment.BlobItems {
|
|
if item.Name == nil {
|
|
continue
|
|
}
|
|
name := strings.TrimPrefix(*item.Name, b.pathPrefix)
|
|
name = strings.TrimPrefix(name, "/")
|
|
entries = append(entries, name)
|
|
}
|
|
}
|
|
return entries, nil
|
|
}
|
|
|
|
func (b *AzureFileBackend) RemoveDirectory(p string) error {
|
|
files, err := b.ListDirectoryRecursively(p)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
for _, f := range files {
|
|
if err := b.RemoveFile(f); err != nil {
|
|
return err
|
|
}
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func (b *AzureFileBackend) ZipReader(p string, deflate bool) (io.ReadCloser, error) {
|
|
method := zip.Store
|
|
if deflate {
|
|
method = zip.Deflate
|
|
}
|
|
|
|
pr, pw := io.Pipe()
|
|
go func() {
|
|
zw := zip.NewWriter(pw)
|
|
err := b.writeZip(zw, p, method)
|
|
if cerr := zw.Close(); err == nil {
|
|
err = cerr
|
|
}
|
|
pw.CloseWithError(err)
|
|
}()
|
|
return pr, nil
|
|
}
|
|
|
|
func (b *AzureFileBackend) writeZip(zw *zip.Writer, p string, method uint16) error {
|
|
exists, err := b.FileExists(p)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
if exists {
|
|
return b.writeZipEntry(zw, p, path.Base(p), method)
|
|
}
|
|
|
|
files, err := b.ListDirectoryRecursively(p)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
prefix := strings.TrimSuffix(p, "/") + "/"
|
|
for _, f := range files {
|
|
rel := strings.TrimPrefix(f, prefix)
|
|
if err := b.writeZipEntry(zw, f, rel, method); err != nil {
|
|
return err
|
|
}
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func (b *AzureFileBackend) writeZipEntry(zw *zip.Writer, blobPath, name string, method uint16) error {
|
|
r, err := b.Reader(blobPath)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
defer r.Close()
|
|
header := &zip.FileHeader{Name: name, Method: method}
|
|
header.SetMode(0644)
|
|
w, err := zw.CreateHeader(header)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
_, err = io.Copy(w, r)
|
|
return err
|
|
}
|
|
|
|
// readSeekNopCloser adapts a Reader+Seeker into a ReadSeekCloser without
|
|
// closing the underlying source. The Azure SDK's StageBlock signature
|
|
// requires a ReadSeekCloser.
|
|
type readSeekNopCloser struct {
|
|
io.Reader
|
|
}
|
|
|
|
func (r *readSeekNopCloser) Seek(offset int64, whence int) (int64, error) {
|
|
return r.Reader.(io.Seeker).Seek(offset, whence)
|
|
}
|
|
|
|
func (r *readSeekNopCloser) Close() error { return nil }
|
|
|
|
// newAzureBlockID returns a fresh base64-encoded 16-byte random block ID,
|
|
// generated with github.com/google/uuid - the same library azblob uses
|
|
// internally for the block IDs it produces in UploadStream. All committed
|
|
// blocks in a single blob must share the same decoded length, so callers
|
|
// must use this for both WriteFile and AppendFile staging.
|
|
//
|
|
// Per https://learn.microsoft.com/en-us/rest/api/storageservices/put-block:
|
|
//
|
|
// For a given blob, all block IDs must be the same length. If a block is
|
|
// uploaded with a block ID of a different length than the block IDs for any
|
|
// existing uncommitted blocks, the service returns error response code 400
|
|
// (Bad Request).
|
|
func newAzureBlockID() (string, error) {
|
|
u, err := uuid.NewRandom()
|
|
if err != nil {
|
|
return "", err
|
|
}
|
|
return base64.StdEncoding.EncodeToString(u[:]), nil
|
|
}
|
|
|
|
func isAzureAuthError(err error) bool {
|
|
if err == nil {
|
|
return false
|
|
}
|
|
return bloberror.HasCode(err, bloberror.AuthenticationFailed) ||
|
|
bloberror.HasCode(err, bloberror.AuthorizationFailure) ||
|
|
bloberror.HasCode(err, bloberror.InvalidAuthenticationInfo)
|
|
}
|