Files
zeekay 8feb24ea1f One name for this fork, and it is ours
The server module named github.com/hanzoai/team/server/v8 — a path that
resolves to the retired workspace's repository, not this one — while
server/public still published github.com/mattermost/mattermost/server/public.
Both are github.com/hanzoteam/server/... now, which is where this code
actually lives.

That second half is what made the binary unbootable. Upstream's public path
stayed claimed by upstream, mattermost-plugin-ai imported it, and the build
linked public/model twice; public/plugin's gob.Register then ran twice under
one name and panicked at init, before main. Reverting the rename traded the
brand for a working binary. Rebranding the plugin alongside it needs no such
trade: the agents fork is github.com/hanzoteam/agents/v2 against our public
module, so no module named mattermost/mattermost is in the graph at all.

Measured: `go list -deps ./cmd/mattermost` names exactly one public/model, and
it is ours.
2026-08-28 23:35:36 -07:00

830 lines
29 KiB
Go

// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
// See LICENSE.txt for license information.
package filestore
import (
"archive/zip"
"bytes"
"context"
"crypto/tls"
"encoding/base64"
"errors"
"fmt"
"io"
"net/http"
"net/url"
"path"
"strings"
"time"
"github.com/Azure/azure-sdk-for-go/sdk/azcore"
"github.com/Azure/azure-sdk-for-go/sdk/azidentity"
"github.com/Azure/azure-sdk-for-go/sdk/storage/azblob"
"github.com/Azure/azure-sdk-for-go/sdk/storage/azblob/blob"
"github.com/Azure/azure-sdk-for-go/sdk/storage/azblob/bloberror"
"github.com/Azure/azure-sdk-for-go/sdk/storage/azblob/blockblob"
"github.com/Azure/azure-sdk-for-go/sdk/storage/azblob/container"
"github.com/Azure/azure-sdk-for-go/sdk/storage/azblob/sas"
"github.com/Azure/azure-sdk-for-go/sdk/storage/azblob/service"
"github.com/google/uuid"
"github.com/hanzoteam/server/server/public/model"
"github.com/hanzoteam/server/server/public/shared/httpservice"
"github.com/hanzoteam/server/server/public/shared/mlog"
)
// azureBlockSize is the chunk size used when staging block blob uploads.
// Matches the Azure SDK's default block size for UploadStream and keeps each
// StageBlock call well under the per-block REST limit (4000 MiB).
const azureBlockSize = 4 * 1024 * 1024
// AzureFileBackend stores files in Azure Blob Storage. Two authentication
// modes are supported: shared key (an account access key configured by the
// admin) and Microsoft Entra ID via DefaultAzureCredential (managed identity,
// service principal, workload identity, or az login - whichever the host
// environment provides).
type AzureFileBackend struct {
client *azblob.Client
// sharedKey holds the account credential when authMode is shared key, and is
// nil under default credential. GeneratePublicLink needs it to sign a Service
// SAS by hand, since GetSASURL does not allow to change the ContentDisposition
// if not at the blob level, and the parsing it does of the URL to retrieve
// container and blob names does not support custom endpoints.
sharedKey *azblob.SharedKeyCredential
authMode string
container string
pathPrefix string
timeout time.Duration
presignExpires time.Duration
ssl bool
}
var _ FileBackendWithLinkGenerator = (*AzureFileBackend)(nil)
func NewAzureFileBackend(settings FileBackendSettings) (*AzureFileBackend, error) {
if err := settings.CheckMandatoryAzureFields(); err != nil {
return nil, err
}
scheme := "https"
if !settings.AzureSSL {
scheme = "http"
}
serviceURL, err := buildAzureServiceURL(settings.AzureCloud, scheme, settings.AzureStorageAccount, settings.AzureEndpoint)
if err != nil {
return nil, err
}
client, sharedKey, err := newAzureClient(settings, serviceURL, buildAzureClientOptions(settings))
if err != nil {
return nil, err
}
// Config.IsValid rejects non-positive timeouts before they reach this
// constructor, but direct callers (tests, library users that build a
// FileBackendSettings by hand) can still slip a zero or negative value
// in. Fall back to a sane default in that case, and log loudly enough
// for the substitution to show up if it ever happens in production.
timeout := time.Duration(settings.AzureRequestTimeoutMilliseconds) * time.Millisecond
if timeout <= 0 {
mlog.Warn("AzureRequestTimeoutMilliseconds is non-positive; falling back to 30s default",
mlog.Int("value", int(settings.AzureRequestTimeoutMilliseconds)))
timeout = 30 * time.Second
}
return &AzureFileBackend{
client: client,
sharedKey: sharedKey,
authMode: settings.AzureAuthMode,
container: settings.AzureContainer,
pathPrefix: settings.AzurePathPrefix,
timeout: timeout,
presignExpires: time.Duration(settings.AzurePresignExpiresSeconds) * time.Second,
ssl: settings.AzureSSL,
}, nil
}
func (b *AzureFileBackend) DriverName() string {
return driverAzure
}
// newAzureClient builds an azblob client for the configured authentication
// mode. Shared key uses NewClientWithSharedKeyCredential; default credential
// uses NewClient with DefaultAzureCredential, which discovers managed
// identity, workload identity, service principal env vars, and az login in
// that order at runtime.
//
// The shared-key credential is returned alongside the client when shared-key
// auth is in use, so callers (GeneratePublicLink in particular) can sign
// Service SAS tokens without round-tripping to Entra ID. It is nil in the
// default-credential path; that path obtains a user-delegation credential
// lazily at link-generation time.
func newAzureClient(settings FileBackendSettings, serviceURL string, clientOptions *azblob.ClientOptions) (*azblob.Client, *azblob.SharedKeyCredential, error) {
switch settings.AzureAuthMode {
case model.AzureAuthModeDefaultCredential:
cred, err := azidentity.NewDefaultAzureCredential(nil)
if err != nil {
return nil, nil, fmt.Errorf("failed to create azure default credential: %w", err)
}
client, err := azblob.NewClient(serviceURL, cred, clientOptions)
if err != nil {
return nil, nil, fmt.Errorf("failed to create azure blob client: %w", err)
}
return client, nil, nil
case model.AzureAuthModeSharedKey:
cred, err := azblob.NewSharedKeyCredential(settings.AzureStorageAccount, settings.AzureAccessKey)
if err != nil {
return nil, nil, fmt.Errorf("failed to create azure shared key credential: %w", err)
}
client, err := azblob.NewClientWithSharedKeyCredential(serviceURL, cred, clientOptions)
if err != nil {
return nil, nil, fmt.Errorf("failed to create azure blob client: %w", err)
}
return client, cred, nil
default:
return nil, nil, fmt.Errorf("unknown azure auth mode %q", settings.AzureAuthMode)
}
}
// buildAzureClientOptions selects the HTTP transport for the Azure SDK. The
// custom cloud routes requests through httpservice's transport, honoring
// AllowedUntrustedInternalConnections. The commercial and government clouds use
// a fixed Azure host (which may resolve to a private address under Private
// Link), so they keep the SDK default transport and only override it to honor
// SkipVerify, mirroring the S3 backend.
func buildAzureClientOptions(settings FileBackendSettings) *azblob.ClientOptions {
if settings.AzureCloud == model.AzureCloudCustom {
return &azblob.ClientOptions{
ClientOptions: azcore.ClientOptions{
Transport: &http.Client{
Transport: httpservice.NewTransportForInternalConnections(settings.SkipVerify, settings.AllowedUntrustedInternalConnections),
},
},
}
}
if settings.SkipVerify {
return &azblob.ClientOptions{
ClientOptions: azcore.ClientOptions{
Transport: &http.Client{
Transport: &http.Transport{
TLSClientConfig: &tls.Config{InsecureSkipVerify: true},
},
},
},
}
}
return nil
}
// buildAzureServiceURL renders the Blob service URL that the SDK signs
// requests against. The cloud value selects the topology:
//
// - commercial -> vhost-style against blob.core.windows.net, e.g.
// https://{account}.blob.core.windows.net/.
// - government -> vhost-style against blob.core.usgovcloudapi.net,
// e.g. https://{account}.blob.core.usgovcloudapi.net/.
// - custom -> the admin-provided endpoint is the full service URL,
// including scheme and storage account (vhost-style for production
// Azure, path-style for Azurite or reverse proxies). Hanzo Team
// does not modify the URL.
//
// Empty cloud is treated as commercial so existing configs that pre-date
// this field keep working. Shared-key auth signs against the URL host,
// so for custom deployments the admin is responsible for ensuring the
// host actually serves the storage account named in the credential.
func buildAzureServiceURL(cloud, scheme, account, endpoint string) (string, error) {
switch cloud {
case model.AzureCloudCommercial, "":
if !model.IsValidAzureStorageAccountName(account) {
return "", fmt.Errorf("invalid azure storage account name %q", account)
}
return fmt.Sprintf("%s://%s.blob.core.windows.net/", scheme, account), nil
case model.AzureCloudGovernment:
if !model.IsValidAzureStorageAccountName(account) {
return "", fmt.Errorf("invalid azure storage account name %q", account)
}
return fmt.Sprintf("%s://%s.blob.core.usgovcloudapi.net/", scheme, account), nil
case model.AzureCloudCustom:
if endpoint == "" {
return "", errors.New("AzureCloud=custom requires AzureEndpoint to be set")
}
// The admin owns this URL end to end, but we still reject inputs
// that the SDK is guaranteed to fail on later (missing scheme,
// missing host, a scheme other than http/https) so the
// failure mode is a clear configuration error at startup rather
// than an opaque SDK error on the first blob request.
parsed, err := url.Parse(endpoint)
if err != nil {
return "", fmt.Errorf("AzureEndpoint is not a valid URL: %w", err)
}
if parsed.Scheme != "http" && parsed.Scheme != "https" {
return "", fmt.Errorf("AzureEndpoint must use http or https, got %q", endpoint)
}
if parsed.Host == "" {
return "", fmt.Errorf("AzureEndpoint must include a host, got %q", endpoint)
}
return endpoint, nil
default:
return "", fmt.Errorf("unknown AzureCloud value %q", cloud)
}
}
// prefix joins the configured pathPrefix and the caller-supplied path.
// Using a plain path.Join, a value like "foo/../../secret" can escape
// the prefix entirely, so we compute the join and verify the result is
// the prefix directory itself or a descendant of it. The descendant check
// requires a path-separator boundary so a prefix of "mattermost" does not
// match a sibling like "mattermost-evil/...". If the joined path escapes,
// we fall back to joining the prefix with path.Base, which may drop any
// intermediate directories the caller intended.
func (b *AzureFileBackend) prefix(p string) string {
joined := path.Join(b.pathPrefix, p)
if b.pathPrefix == "" {
return joined
}
cleanPrefix := strings.TrimSuffix(path.Clean(b.pathPrefix), "/")
if joined == cleanPrefix || strings.HasPrefix(joined, cleanPrefix+"/") {
return joined
}
return path.Join(cleanPrefix, path.Base(p))
}
func (b *AzureFileBackend) newBlobClient(p string) *blob.Client {
return b.client.ServiceClient().NewContainerClient(b.container).NewBlobClient(b.prefix(p))
}
func (b *AzureFileBackend) newBlockBlobClient(p string) *blockblob.Client {
return b.client.ServiceClient().NewContainerClient(b.container).NewBlockBlobClient(b.prefix(p))
}
func (b *AzureFileBackend) newContainerClient() *container.Client {
return b.client.ServiceClient().NewContainerClient(b.container)
}
// TestConnection probes the configured container and reports the outcome
// using the typed errors shared with the other backends. Container
// creation is deliberately out of scope here - callers (Server.Start)
// decide whether to provision a missing container via MakeContainer.
// That separation keeps a typo in the System Console from silently
// provisioning an unwanted container, and matches the S3 contract where
// TestConnection returns FileBackendNoBucketError and MakeBucket is an
// explicit call.
func (b *AzureFileBackend) TestConnection() error {
ctx, cancel := context.WithTimeout(context.Background(), b.timeout)
defer cancel()
_, err := b.newContainerClient().GetProperties(ctx, nil)
if err == nil {
return nil
}
if bloberror.HasCode(err, bloberror.ContainerNotFound) {
return &FileBackendNoBucketError{Err: fmt.Errorf("azure container %q does not exist: %w", b.container, err)}
}
if isAzureAuthError(err) {
return &FileBackendAuthError{Err: fmt.Errorf("unable to authenticate against azure blob storage: %w", err)}
}
return fmt.Errorf("unable to connect to azure blob storage: %w", err)
}
// MakeContainer creates the configured container. Mirrors S3FileBackend.MakeBucket
// so callers can opt into container provisioning explicitly. An already-existing
// container is treated as success so that concurrent boots (two nodes racing
// through TestConnection plus MakeContainer) both converge cleanly.
func (b *AzureFileBackend) MakeContainer() error {
ctx, cancel := context.WithTimeout(context.Background(), b.timeout)
defer cancel()
if _, err := b.newContainerClient().Create(ctx, nil); err != nil {
if bloberror.HasCode(err, bloberror.ContainerAlreadyExists) {
return nil
}
return fmt.Errorf("unable to create azure container %q: %w", b.container, err)
}
return nil
}
func (b *AzureFileBackend) Reader(p string) (ReadCloseSeeker, error) {
// Arm the deadline *before* the first network call, then hand the same
// timer to the returned reader on success. The previous code only set up
// the timer on the happy path, which left GetProperties running against a
// no-deadline context.
ctx, cancel := context.WithCancel(context.Background())
timer := time.AfterFunc(b.timeout, cancel)
blobClient := b.newBlobClient(p)
props, err := blobClient.GetProperties(ctx, nil)
if err != nil {
timer.Stop()
cancel()
return nil, fmt.Errorf("unable to read file %q: %w", p, err)
}
if props.ContentLength == nil {
timer.Stop()
cancel()
return nil, fmt.Errorf("missing content length for %q", p)
}
return &azureRangeReader{
ctx: ctx,
cancel: cancel,
timer: timer,
blobClient: blobClient,
size: *props.ContentLength,
}, nil
}
func (b *AzureFileBackend) ReadFile(p string) ([]byte, error) {
r, err := b.Reader(p)
if err != nil {
return nil, err
}
defer r.Close()
return io.ReadAll(r)
}
func (b *AzureFileBackend) FileExists(p string) (bool, error) {
ctx, cancel := context.WithTimeout(context.Background(), b.timeout)
defer cancel()
_, err := b.newBlobClient(p).GetProperties(ctx, nil)
if err != nil {
if bloberror.HasCode(err, bloberror.BlobNotFound) {
return false, nil
}
return false, fmt.Errorf("unable to check existence of %q: %w", p, err)
}
return true, nil
}
func (b *AzureFileBackend) FileSize(p string) (int64, error) {
ctx, cancel := context.WithTimeout(context.Background(), b.timeout)
defer cancel()
props, err := b.newBlobClient(p).GetProperties(ctx, nil)
if err != nil {
return 0, fmt.Errorf("unable to get size of %q: %w", p, err)
}
return model.SafeDereference(props.ContentLength), nil
}
func (b *AzureFileBackend) FileModTime(p string) (time.Time, error) {
ctx, cancel := context.WithTimeout(context.Background(), b.timeout)
defer cancel()
props, err := b.newBlobClient(p).GetProperties(ctx, nil)
if err != nil {
return time.Time{}, fmt.Errorf("unable to get modification time of %q: %w", p, err)
}
return model.SafeDereference(props.LastModified), nil
}
// CopyFile copies via StartCopyFromURL and polls the resulting blob's copy
// status until it succeeds, matching the synchronous semantics that the
// FileBackend interface (and the S3 driver via ComposeObject) provides.
func (b *AzureFileBackend) CopyFile(oldPath, newPath string) error {
ctx, cancel := context.WithTimeout(context.Background(), b.timeout)
defer cancel()
src := b.newBlobClient(oldPath).URL()
dst := b.newBlockBlobClient(newPath)
if _, err := dst.StartCopyFromURL(ctx, src, nil); err != nil {
return fmt.Errorf("unable to copy %q to %q: %w", oldPath, newPath, err)
}
// Poll until the copy reports success. For server-to-server copies within
// the same account this is typically synchronous, but the API is
// asynchronous in general, so we wait.
for {
props, err := dst.GetProperties(ctx, nil)
if err != nil {
return fmt.Errorf("unable to read copy status for %q: %w", newPath, err)
}
if props.CopyStatus == nil {
return nil
}
switch *props.CopyStatus {
case blob.CopyStatusTypeSuccess:
return nil
case blob.CopyStatusTypeFailed, blob.CopyStatusTypeAborted:
desc := model.SafeDereference(props.CopyStatusDescription)
return fmt.Errorf("azure copy from %q to %q ended in status %q: %q", oldPath, newPath, *props.CopyStatus, desc)
}
select {
case <-ctx.Done():
return fmt.Errorf("azure copy from %q to %q did not complete in time: %w", oldPath, newPath, ctx.Err())
case <-time.After(50 * time.Millisecond):
}
}
}
func (b *AzureFileBackend) MoveFile(oldPath, newPath string) error {
if err := b.CopyFile(oldPath, newPath); err != nil {
return err
}
return b.RemoveFile(oldPath)
}
func (b *AzureFileBackend) WriteFile(fr io.Reader, p string) (int64, error) {
ctx, cancel := context.WithTimeout(context.Background(), b.timeout)
defer cancel()
return b.WriteFileContext(ctx, fr, p)
}
// stageBlocks reads fr in azureBlockSize chunks and stages each chunk as a
// block under a fresh ID. Returns the IDs of the newly staged blocks (in
// order) and the total byte count. The caller is responsible for committing
// the block list.
func (b *AzureFileBackend) stageBlocks(ctx context.Context, bb *blockblob.Client, fr io.Reader, p string) ([]string, int64, error) {
buf := make([]byte, azureBlockSize)
var ids []string
var total int64
for {
n, err := io.ReadFull(fr, buf)
if n > 0 {
id, idErr := newAzureBlockID()
if idErr != nil {
return nil, 0, fmt.Errorf("failed to generate azure block id: %w", idErr)
}
if _, sbErr := bb.StageBlock(ctx, id, &readSeekNopCloser{Reader: bytes.NewReader(buf[:n])}, nil); sbErr != nil {
return nil, 0, fmt.Errorf("unable to stage block for %q: %w", p, sbErr)
}
ids = append(ids, id)
total += int64(n)
}
if err == io.EOF || err == io.ErrUnexpectedEOF {
break
}
if err != nil {
return nil, 0, fmt.Errorf("failed to read input: %w", err)
}
}
return ids, total, nil
}
// WriteFileContext stages the body in fixed-size blocks and commits a fresh
// block list. It deliberately does not use the SDK's UploadStream helper:
// UploadStream's small-payload fast path falls back to single-shot PutBlob,
// which leaves the resulting blob with no committed block list. A subsequent
// AppendFile that calls CommitBlockList on that blob would then clobber its
// content. Routing every WriteFile through StageBlock + CommitBlockList keeps
// AppendFile correct regardless of payload size.
//
// The caller's context governs the entire upload - no inner timeout is added.
// TryWriteFileContext (filesstore.go) relies on this to let long-running
// callers like message-export bulk writes opt out of the per-operation
// timeout that WriteFile applies by default.
func (b *AzureFileBackend) WriteFileContext(ctx context.Context, fr io.Reader, p string) (int64, error) {
bb := b.newBlockBlobClient(p)
blockIDs, total, err := b.stageBlocks(ctx, bb, fr, p)
if err != nil {
return 0, err
}
if len(blockIDs) == 0 {
// Empty input - still need to materialize an empty blob with a
// committed block list so AppendFile can target it.
id, idErr := newAzureBlockID()
if idErr != nil {
return 0, fmt.Errorf("failed to generate azure block id: %w", idErr)
}
if _, sbErr := bb.StageBlock(ctx, id, &readSeekNopCloser{Reader: bytes.NewReader(nil)}, nil); sbErr != nil {
return 0, fmt.Errorf("unable to stage empty block for %q: %w", p, sbErr)
}
blockIDs = append(blockIDs, id)
}
if _, err := bb.CommitBlockList(ctx, blockIDs, nil); err != nil {
return 0, fmt.Errorf("unable to commit block list for %q: %w", p, err)
}
return total, nil
}
// AppendFile stages the new chunk as one or more blocks and commits the
// existing committed block list plus the newly staged IDs. Each AppendFile
// call uploads the new bytes exactly once - no re-download, no
// re-concatenate, no re-upload of the prior contents. The S3-style contract
// is preserved: returns an error if the target blob does not yet exist;
// returns the number of bytes appended (not the resulting total size).
//
// Refuses to append to a blob that has content but no committed block list
// (i.e. was uploaded via Put Blob by another tool - Azure portal, azcopy,
// a migration script). Committing a new block list against such a blob
// would replace the existing content with only the appended bytes, so
// failing loud beats silent data loss.
func (b *AzureFileBackend) AppendFile(fr io.Reader, p string) (int64, error) {
ctx, cancel := context.WithTimeout(context.Background(), b.timeout)
defer cancel()
bb := b.newBlockBlobClient(p)
listResp, err := bb.GetBlockList(ctx, blockblob.BlockListTypeCommitted, nil)
if err != nil {
return 0, fmt.Errorf("unable to find file %q to append data: %w", p, err)
}
var existingIDs []string
if listResp.BlockList.CommittedBlocks != nil {
for _, blk := range listResp.BlockList.CommittedBlocks {
if blk.Name != nil {
existingIDs = append(existingIDs, *blk.Name)
}
}
}
if len(existingIDs) == 0 {
props, propsErr := bb.GetProperties(ctx, nil)
if propsErr != nil {
return 0, fmt.Errorf("unable to inspect %q before append: %w", p, propsErr)
}
if model.SafeDereference(props.ContentLength) > 0 {
return 0, fmt.Errorf("refusing to append to %q: blob has content but no committed block list (likely written via Put Blob by another tool)", p)
}
}
newIDs, total, err := b.stageBlocks(ctx, bb, fr, p)
if err != nil {
return 0, err
}
if _, err := bb.CommitBlockList(ctx, append(existingIDs, newIDs...), nil); err != nil {
return 0, fmt.Errorf("unable to commit block list for %q: %w", p, err)
}
return total, nil
}
// GeneratePublicLink returns a time-limited, read-only URL to the blob at path
// using a Shared Access Signature. The SAS is auth-mode aware:
//
// - shared-key auth signs a Service SAS in-process with the stored
// SharedKeyCredential.
// - default-credential auth fetches a user-delegation key from Entra ID
// (one round trip per call) and signs a user-delegation SAS with it.
//
// This is intended for the export-download flow (App.GeneratePresignURLForExport
// and the /exportlink slash command). End users never reach this code path.
func (b *AzureFileBackend) GeneratePublicLink(path string) (string, time.Duration, error) {
if b.presignExpires <= 0 {
return "", 0, errors.New("azure presign expiration is not configured")
}
prefixed := b.prefix(path)
// Back-date the start by a small fixed amount to absorb minor clock skew
// between this host and Azure, matching the azure-sdk-for-go SAS examples:
// https://github.com/Azure/azure-sdk-for-go/blob/65c3b792856d9ad7ce0b59c127ce299358e41a01/sdk/storage/azblob/sas/examples_test.go#L71
// https://github.com/Azure/azure-sdk-for-go/blob/65c3b792856d9ad7ce0b59c127ce299358e41a01/sdk/storage/azblob/service/examples_test.go#L315
const clockSkew = 10 * time.Second
start := time.Now().UTC().Add(-clockSkew)
expiry := start.Add(b.presignExpires)
protocol := sas.ProtocolHTTPSandHTTP
if b.ssl {
protocol = sas.ProtocolHTTPS
}
values := sas.BlobSignatureValues{
Protocol: protocol,
StartTime: start,
ExpiryTime: expiry,
Permissions: (&sas.BlobPermissions{Read: true}).String(),
ContainerName: b.container,
BlobName: prefixed,
// Request browsers to download the file instead of rendering it
ContentDisposition: "attachment",
}
var (
qps sas.QueryParameters
err error
)
switch b.authMode {
case model.AzureAuthModeSharedKey:
if b.sharedKey == nil {
return "", 0, errors.New("shared key credential is unavailable")
}
qps, err = values.SignWithSharedKey(b.sharedKey)
if err != nil {
return "", 0, fmt.Errorf("unable to sign service SAS for %q: %w", path, err)
}
case model.AzureAuthModeDefaultCredential:
ctx, cancel := context.WithTimeout(context.Background(), b.timeout)
defer cancel()
udc, udcErr := b.client.ServiceClient().GetUserDelegationCredential(ctx, service.KeyInfo{
Start: new(start.Format(sas.TimeFormat)),
Expiry: new(expiry.Format(sas.TimeFormat)),
}, nil)
if udcErr != nil {
return "", 0, fmt.Errorf("unable to obtain user delegation key for %q: %w", path, udcErr)
}
qps, err = values.SignWithUserDelegation(udc)
if err != nil {
return "", 0, fmt.Errorf("unable to sign user-delegation SAS for %q: %w", path, err)
}
default:
return "", 0, fmt.Errorf("unknown azure auth mode %q", b.authMode)
}
return b.newBlobClient(path).URL() + "?" + qps.Encode(), b.presignExpires, nil
}
func (b *AzureFileBackend) RemoveFile(p string) error {
ctx, cancel := context.WithTimeout(context.Background(), b.timeout)
defer cancel()
_, err := b.newBlobClient(p).Delete(ctx, nil)
if err != nil && !bloberror.HasCode(err, bloberror.BlobNotFound) {
return fmt.Errorf("unable to remove file %q: %w", p, err)
}
return nil
}
func (b *AzureFileBackend) ListDirectory(p string) ([]string, error) {
prefix := b.prefix(p)
if prefix != "" && !strings.HasSuffix(prefix, "/") {
prefix += "/"
}
ctx, cancel := context.WithTimeout(context.Background(), b.timeout)
defer cancel()
pager := b.newContainerClient().NewListBlobsHierarchyPager("/", &container.ListBlobsHierarchyOptions{
Prefix: &prefix,
})
var entries []string
for pager.More() {
page, err := pager.NextPage(ctx)
if err != nil {
return nil, fmt.Errorf("unable to list directory %q: %w", p, err)
}
for _, item := range page.Segment.BlobItems {
if item.Name == nil {
continue
}
name := strings.TrimPrefix(*item.Name, b.pathPrefix)
name = strings.TrimPrefix(name, "/")
entries = append(entries, name)
}
for _, item := range page.Segment.BlobPrefixes {
if item.Name == nil {
continue
}
name := strings.TrimPrefix(*item.Name, b.pathPrefix)
name = strings.TrimPrefix(name, "/")
name = strings.TrimSuffix(name, "/")
entries = append(entries, name)
}
}
return entries, nil
}
func (b *AzureFileBackend) ListDirectoryRecursively(p string) ([]string, error) {
prefix := b.prefix(p)
if prefix != "" && !strings.HasSuffix(prefix, "/") {
prefix += "/"
}
ctx, cancel := context.WithTimeout(context.Background(), b.timeout)
defer cancel()
pager := b.newContainerClient().NewListBlobsFlatPager(&container.ListBlobsFlatOptions{
Prefix: &prefix,
})
var entries []string
for pager.More() {
page, err := pager.NextPage(ctx)
if err != nil {
return nil, fmt.Errorf("unable to list directory %q recursively: %w", p, err)
}
for _, item := range page.Segment.BlobItems {
if item.Name == nil {
continue
}
name := strings.TrimPrefix(*item.Name, b.pathPrefix)
name = strings.TrimPrefix(name, "/")
entries = append(entries, name)
}
}
return entries, nil
}
func (b *AzureFileBackend) RemoveDirectory(p string) error {
files, err := b.ListDirectoryRecursively(p)
if err != nil {
return err
}
for _, f := range files {
if err := b.RemoveFile(f); err != nil {
return err
}
}
return nil
}
func (b *AzureFileBackend) ZipReader(p string, deflate bool) (io.ReadCloser, error) {
method := zip.Store
if deflate {
method = zip.Deflate
}
pr, pw := io.Pipe()
go func() {
zw := zip.NewWriter(pw)
err := b.writeZip(zw, p, method)
if cerr := zw.Close(); err == nil {
err = cerr
}
pw.CloseWithError(err)
}()
return pr, nil
}
func (b *AzureFileBackend) writeZip(zw *zip.Writer, p string, method uint16) error {
exists, err := b.FileExists(p)
if err != nil {
return err
}
if exists {
return b.writeZipEntry(zw, p, path.Base(p), method)
}
files, err := b.ListDirectoryRecursively(p)
if err != nil {
return err
}
prefix := strings.TrimSuffix(p, "/") + "/"
for _, f := range files {
rel := strings.TrimPrefix(f, prefix)
if err := b.writeZipEntry(zw, f, rel, method); err != nil {
return err
}
}
return nil
}
func (b *AzureFileBackend) writeZipEntry(zw *zip.Writer, blobPath, name string, method uint16) error {
r, err := b.Reader(blobPath)
if err != nil {
return err
}
defer r.Close()
header := &zip.FileHeader{Name: name, Method: method}
header.SetMode(0644)
w, err := zw.CreateHeader(header)
if err != nil {
return err
}
_, err = io.Copy(w, r)
return err
}
// readSeekNopCloser adapts a Reader+Seeker into a ReadSeekCloser without
// closing the underlying source. The Azure SDK's StageBlock signature
// requires a ReadSeekCloser.
type readSeekNopCloser struct {
io.Reader
}
func (r *readSeekNopCloser) Seek(offset int64, whence int) (int64, error) {
return r.Reader.(io.Seeker).Seek(offset, whence)
}
func (r *readSeekNopCloser) Close() error { return nil }
// newAzureBlockID returns a fresh base64-encoded 16-byte random block ID,
// generated with github.com/google/uuid - the same library azblob uses
// internally for the block IDs it produces in UploadStream. All committed
// blocks in a single blob must share the same decoded length, so callers
// must use this for both WriteFile and AppendFile staging.
//
// Per https://learn.microsoft.com/en-us/rest/api/storageservices/put-block:
//
// For a given blob, all block IDs must be the same length. If a block is
// uploaded with a block ID of a different length than the block IDs for any
// existing uncommitted blocks, the service returns error response code 400
// (Bad Request).
func newAzureBlockID() (string, error) {
u, err := uuid.NewRandom()
if err != nil {
return "", err
}
return base64.StdEncoding.EncodeToString(u[:]), nil
}
func isAzureAuthError(err error) bool {
if err == nil {
return false
}
return bloberror.HasCode(err, bloberror.AuthenticationFailed) ||
bloberror.HasCode(err, bloberror.AuthorizationFailure) ||
bloberror.HasCode(err, bloberror.InvalidAuthenticationInfo)
}