mirror of
https://github.com/hanzoai/skills.git
synced 2026-11-19 00:47:36 +00:00
Every page now loads /webmcp.js, scripts/webmcp.ts with its types stripped (npm run webmcp). Where the browser has document.modelContext it registers two read-only tools: search_skills over the published skills index, and get_skill, which returns a skill's markdown checked against the index digest. The markdown alternate link moves from build.py to site.py, so CI holds every page to it. The API catalog gains the site itself (llms.txt and the skills index) and hanzo.ai/agents; the live discovery check now fetches every catalog link. tests/site.mjs drives the tools through a recording modelContext, and checks a browser without one loads clean.
707 lines
33 KiB
HTML
707 lines
33 KiB
HTML
<!doctype html>
|
|
<html lang="en"><head>
|
|
<meta charset="utf-8"><meta name="viewport" content="width=device-width,initial-scale=1">
|
|
<title>cloud-aws-storage</title>
|
|
<meta name="color-scheme" content="dark">
|
|
<style>
|
|
/* Committed to one visual world: a dark instrument panel. No light variant —
|
|
the dot field and the hairline rules only read on black, and a naive
|
|
inversion would wash both out. Every colour is painted explicitly so the page
|
|
never borrows a host background. */
|
|
@font-face{font-family:'Zen';src:url('/fonts/Zen-Variable.woff2') format('woff2');font-weight:100 900;font-style:normal;font-display:swap}
|
|
@font-face{font-family:'Zen';src:url('/fonts/Zen-Italic-Variable.woff2') format('woff2');font-weight:100 900;font-style:italic;font-display:swap}
|
|
@font-face{font-family:'Zen Mono';src:url('/fonts/ZenMono-Variable.woff2') format('woff2');font-weight:100 900;font-style:normal;font-display:swap}
|
|
@font-face{font-family:'Zen Mono';src:url('/fonts/ZenMono-Italic-Variable.woff2') format('woff2');font-weight:100 900;font-style:italic;font-display:swap}
|
|
@font-face{font-family:'Zen Pixel Square';src:url('/fonts/ZenPixel-Square.woff2') format('woff2');font-style:normal;font-display:swap}
|
|
|
|
:root{
|
|
--ink:#f4f4f2; --soft:#9c9c98; --faint:#6a6a66; --ground:#000; --panel:#0a0a0a;
|
|
--line:#232323; --line-hi:#3a3a3a; --accent:#f4f4f2;
|
|
--font-zen-sans:'Zen',-apple-system,BlinkMacSystemFont,"Segoe UI",Helvetica,Arial,sans-serif;
|
|
--font-zen-mono:'Zen Mono',ui-monospace,"SF Mono",SFMono-Regular,Menlo,Consolas,monospace;
|
|
--font-zen-pixel-square:'Zen Pixel Square',monospace;
|
|
--mono:var(--font-zen-mono);
|
|
--sans:var(--font-zen-sans);
|
|
/* Controls are pills and surfaces are soft, as on hanzo.ai. */
|
|
--r:999px; --r-soft:14px;
|
|
}
|
|
*{box-sizing:border-box}
|
|
html{-webkit-text-size-adjust:100%}
|
|
body{margin:0;background:var(--ground);color:var(--ink);font-family:var(--sans);
|
|
font-size:16px;line-height:1.65;-webkit-font-smoothing:antialiased}
|
|
.wrap{max-width:60rem;margin:0 auto;padding:0 1.75rem}
|
|
|
|
/* The dot field. A halftone made of one repeating radial-gradient, faded out
|
|
along the diagonal by a mask so it reads as a wave rather than wallpaper. */
|
|
/* No overflow clip here: the dot field is inset to the box already, and a clip
|
|
would cut off the copy-prompt menu. */
|
|
.hero{position:relative;border-bottom:1px solid var(--line)}
|
|
.hero::before{content:"";position:absolute;inset:0;
|
|
background-image:radial-gradient(var(--line-hi) 1px, transparent 1px);
|
|
background-size:7px 7px;
|
|
-webkit-mask-image:linear-gradient(115deg,#000 0%,rgba(0,0,0,.55) 38%,transparent 72%);
|
|
mask-image:linear-gradient(115deg,#000 0%,rgba(0,0,0,.55) 38%,transparent 72%);
|
|
pointer-events:none}
|
|
.hero .wrap{position:relative;padding-top:5.5rem;padding-bottom:4rem}
|
|
|
|
nav{border-bottom:1px solid var(--line)}
|
|
nav .in{max-width:60rem;margin:0 auto;padding:1.1rem 1.75rem;display:flex;
|
|
justify-content:space-between;align-items:center;gap:1rem}
|
|
.brand{font-family:var(--mono);font-size:.85rem;letter-spacing:.14em;
|
|
text-transform:uppercase;color:var(--ink);text-decoration:none;
|
|
display:inline-flex;align-items:center;gap:.6rem}
|
|
.brand i{font-style:normal;color:var(--faint)}
|
|
.mark{width:1.15em;height:1.15em;display:block;flex:0 0 auto}
|
|
nav a.up{font-family:var(--mono);font-size:.72rem;letter-spacing:.14em;
|
|
text-transform:uppercase;color:var(--soft);text-decoration:none}
|
|
nav a.up:hover{color:var(--ink)}
|
|
|
|
h1{font-family:var(--mono);font-size:clamp(2rem,5.4vw,3.4rem);line-height:1.12;
|
|
letter-spacing:-.02em;margin:0 0 1.1rem;font-weight:500;text-wrap:balance}
|
|
h1 em{font-style:normal;color:var(--faint)}
|
|
.lede{color:var(--soft);font-size:1.05rem;max-width:52ch;margin:0 0 2.25rem}
|
|
|
|
.btns{display:flex;flex-wrap:wrap;gap:.7rem;margin:0 0 .5rem;align-items:center}
|
|
.btn{display:inline-flex;align-items:center;gap:.5rem;font-family:var(--mono);
|
|
font-size:.72rem;letter-spacing:.14em;text-transform:uppercase;
|
|
padding:.8rem 1.4rem;border:1px solid var(--line-hi);border-radius:var(--r);
|
|
color:var(--ink);text-decoration:none;background:transparent;cursor:pointer;
|
|
line-height:1.2;font-weight:400}
|
|
.btn:hover{background:#141414;text-decoration:none;border-color:var(--soft)}
|
|
.btn.solid{background:var(--ink);color:#000;border-color:var(--ink)}
|
|
.btn.solid:hover{background:#d8d8d4}
|
|
.btn .chev{width:.7em;height:.7em;flex:0 0 auto;transition:transform .15s ease}
|
|
.btn[aria-expanded="true"] .chev{transform:rotate(180deg)}
|
|
|
|
/* Copy-prompt selector. Each row copies the prompt for its own target. */
|
|
.copy{position:relative;display:inline-flex}
|
|
.menu{position:absolute;top:calc(100% + .5rem);left:0;z-index:20;min-width:21rem;
|
|
background:var(--panel);border:1px solid var(--line-hi);border-radius:var(--r-soft);
|
|
padding:.4rem;display:flex;flex-direction:column;
|
|
box-shadow:0 12px 32px rgba(0,0,0,.75)}
|
|
.menu[hidden]{display:none}
|
|
.mi{display:flex;align-items:center;gap:.7rem;width:100%;text-align:left;
|
|
padding:.62rem .8rem;border:0;border-radius:calc(var(--r-soft) - .35rem);
|
|
background:transparent;color:var(--ink);font-family:var(--mono);font-size:.76rem;
|
|
letter-spacing:.04em;text-decoration:none;cursor:pointer;line-height:1.3;
|
|
white-space:nowrap}
|
|
.mi:hover,.mi:focus-visible{background:#181818;outline:none}
|
|
.mi .mark{width:1.05rem;height:1.05rem;color:var(--soft)}
|
|
.mi:hover .mark{color:var(--ink)}
|
|
.mi u{margin-left:auto;color:var(--faint);text-decoration:none;font-size:.68rem;
|
|
letter-spacing:.1em;text-transform:uppercase}
|
|
|
|
/* Works-with bar: one scrolling row, name and mark per agent. */
|
|
.strip{border-bottom:1px solid var(--line);background:var(--panel)}
|
|
.strip .in{max-width:60rem;margin:0 auto;padding:.9rem 1.75rem;display:flex;
|
|
align-items:center;gap:1.25rem;font-family:var(--mono);font-size:.72rem;
|
|
letter-spacing:.1em;text-transform:uppercase;color:var(--faint)}
|
|
.strip b{color:var(--soft);font-weight:400;flex:0 0 auto;
|
|
border-right:1px solid var(--line);padding-right:1.25rem;align-self:stretch;
|
|
display:flex;align-items:center}
|
|
.bar{display:flex;align-items:center;gap:.5rem;flex:1 1 auto;min-width:0;
|
|
overflow-x:auto;scrollbar-width:none;scroll-snap-type:x proximity;
|
|
-webkit-overflow-scrolling:touch;
|
|
-webkit-mask-image:linear-gradient(90deg,#000 calc(100% - 3rem),transparent);
|
|
mask-image:linear-gradient(90deg,#000 calc(100% - 3rem),transparent)}
|
|
.bar::-webkit-scrollbar{display:none}
|
|
.agent{display:inline-flex;align-items:center;gap:.45rem;flex:0 0 auto;
|
|
padding:.4rem .85rem;border:1px solid var(--line);border-radius:var(--r);
|
|
color:var(--soft);white-space:nowrap;scroll-snap-align:start;
|
|
text-decoration:none;background:var(--ground)}
|
|
.agent:hover{color:var(--ink);border-color:var(--line-hi);text-decoration:none}
|
|
.agent .mark{width:.95rem;height:.95rem}
|
|
|
|
h2{font-family:var(--mono);font-size:1.1rem;letter-spacing:-.01em;margin:2.75rem 0 .75rem;font-weight:500}
|
|
h3{font-size:1rem;margin:2rem 0 .5rem;font-weight:600}
|
|
h4,h5,h6{font-size:.94rem;margin:1.5rem 0 .5rem;font-weight:600}
|
|
p{margin:0 0 1rem;max-width:70ch}
|
|
ul,ol{margin:0 0 1rem;padding-left:1.35rem;max-width:70ch}
|
|
li{margin:.22rem 0}
|
|
a{color:var(--ink);text-decoration:underline;text-underline-offset:3px;
|
|
text-decoration-color:var(--line-hi)}
|
|
a:hover{text-decoration-color:var(--ink)}
|
|
a:focus-visible{outline:1px solid var(--ink);outline-offset:3px}
|
|
code{font-family:var(--mono);font-size:.85em;background:var(--panel);
|
|
border:1px solid var(--line);border-radius:5px;padding:.1em .36em}
|
|
pre{background:var(--panel);border:1px solid var(--line);padding:1rem 1.1rem;
|
|
border-radius:var(--r-soft);overflow-x:auto;margin:0 0 1.25rem}
|
|
pre code{background:none;border:0;padding:0;font-size:.82rem;line-height:1.7}
|
|
blockquote{margin:0 0 1rem;padding-left:1rem;border-left:1px solid var(--line-hi);color:var(--soft)}
|
|
hr{border:0;border-top:1px solid var(--line);margin:2.5rem 0}
|
|
table{border-collapse:collapse;width:100%;margin:0 0 1.25rem;font-size:.92rem;
|
|
display:block;overflow-x:auto}
|
|
th,td{border:1px solid var(--line);padding:.5rem .7rem;text-align:left}
|
|
|
|
.curl{display:block;background:var(--panel);border:1px solid var(--line);
|
|
border-radius:var(--r-soft);padding:.95rem 1.1rem;font-family:var(--mono);
|
|
font-size:.8rem;color:var(--soft);overflow-x:auto;margin:0 0 2.5rem;
|
|
white-space:pre;line-height:1.9}
|
|
.curl b{color:var(--ink);font-weight:400}
|
|
.curl i{font-style:normal;color:var(--faint)}
|
|
|
|
/* index */
|
|
#q{width:100%;padding:.85rem 1.2rem;font-family:var(--mono);font-size:.85rem;color:var(--ink);
|
|
background:var(--panel);border:1px solid var(--line);border-radius:var(--r);margin:0 0 .6rem}
|
|
#q::placeholder{color:var(--faint);text-transform:uppercase;letter-spacing:.1em;font-size:.72rem}
|
|
#q:focus{outline:none;border-color:var(--soft)}
|
|
#count{color:var(--faint);font-family:var(--mono);font-size:.72rem;letter-spacing:.1em;
|
|
text-transform:uppercase;margin:0 0 2rem}
|
|
.cat{margin:2.5rem 0 .5rem;font-family:var(--mono);font-size:.7rem;letter-spacing:.16em;
|
|
text-transform:uppercase;color:var(--faint);border-bottom:1px solid var(--line);
|
|
padding-bottom:.45rem}
|
|
.row{display:block;padding:.6rem .9rem;margin:0 -.9rem;border-bottom:1px solid var(--line);
|
|
border-radius:var(--r-soft);text-decoration:none}
|
|
.row:hover{background:#0a0a0a}
|
|
.nm{font-family:var(--mono);font-size:.84rem;color:var(--ink);display:block}
|
|
.ds{color:var(--soft);font-size:.9rem;display:block;margin-top:.12rem}
|
|
footer{margin-top:4rem;padding:2rem 0 4rem;border-top:1px solid var(--line);
|
|
color:var(--faint);font-size:.85rem}
|
|
footer a{color:var(--soft)}
|
|
.fmap{display:grid;gap:1.75rem;grid-template-columns:repeat(auto-fit,minmax(11rem,1fr));
|
|
margin:0 0 2rem;font-size:.85rem}
|
|
.fmap b{display:block;font-family:var(--mono);font-size:.68rem;letter-spacing:.16em;
|
|
text-transform:uppercase;color:var(--faint);font-weight:400;margin:0 0 .6rem;
|
|
border-bottom:1px solid var(--line);padding-bottom:.4rem}
|
|
.fmap a{display:inline-block;color:var(--ink);text-decoration:none;margin:.3rem .35rem 0 0}
|
|
.fmap a:hover{text-decoration:underline}
|
|
.fmap div{color:var(--faint);line-height:1.5}
|
|
.fend{border-top:1px solid var(--line);padding-top:1.25rem;font-size:.82rem}
|
|
|
|
@media (max-width:640px){
|
|
.hero .wrap{padding-top:3.25rem;padding-bottom:2.75rem}
|
|
.btns{gap:.5rem}
|
|
.btn{padding:.75rem 1.15rem;font-size:.68rem}
|
|
.copy,.copy .btn{width:100%;justify-content:center}
|
|
.menu{left:0;right:0;min-width:0}
|
|
.strip .in{flex-direction:column;align-items:flex-start;gap:.6rem;padding:.9rem 1.75rem}
|
|
.strip b{border-right:0;padding-right:0}
|
|
.bar{width:100%}
|
|
}
|
|
</style>
|
|
<link rel="icon" href="/favicon.svg">
|
|
<link rel="canonical" href="https://hanzoskills.com/cloud-aws-storage">
|
|
<link rel="alternate" type="text/markdown" href="/cloud-aws-storage.md">
|
|
<script type="module" src="/webmcp.js"></script>
|
|
</head>
|
|
<!-- Reading this with a machine? Every page here is also plain markdown.
|
|
This one: https://hanzoskills.com/cloud-aws-storage.md
|
|
Index: https://hanzoskills.com/llms.txt
|
|
Corpus: https://hanzoskills.com/all.md
|
|
Append .md to any path for the markdown twin. -->
|
|
<body>
|
|
<nav><div class="in">
|
|
<a class="brand" href="/"><svg class="mark" viewBox="0 0 67 67" fill="currentColor" aria-hidden="true"><path d="M22.21 67V44.6369H0V67H22.21Z"/><path d="M66.7038 22.3184H22.2534L0.0878906 44.6367H44.4634L66.7038 22.3184Z"/><path d="M22.21 0H0V22.3184H22.21V0Z"/><path d="M66.7198 0H44.5098V22.3184H66.7198V0Z"/><path d="M66.7198 67V44.6369H44.5098V67H66.7198Z"/></svg> HANZO<i>SKILLS</i></a>
|
|
<a class="up" href="/">All skills</a>
|
|
</div></nav>
|
|
|
|
<div class="wrap">
|
|
<h1>cloud-aws-storage</h1><p class="lede">AWS storage services - S3, EBS, EFS, Glacier, lifecycle policies, encryption, and data transfer</p><h1>AWS Storage</h1>
|
|
<p><strong>Scope</strong>: AWS storage - S3 buckets, EBS volumes, EFS file systems, Glacier archival, lifecycle policies, encryption <strong>Lines</strong>: ~300 <strong>Last Updated</strong>: 2025-10-25 <strong>Format Version</strong>: 1.0 (Atomic)</p>
|
|
<hr>
|
|
<h2>When to Use This Skill</h2>
|
|
<p>Activate this skill when:</p>
|
|
<ul><li>Storing objects, files, or backups in S3</li><li>Attaching block storage to EC2 with EBS</li><li>Sharing file systems across instances with EFS</li><li>Archiving data long-term with Glacier</li><li>Configuring S3 lifecycle policies for cost optimization</li><li>Setting up S3 event notifications for processing</li><li>Implementing encryption at rest and in transit</li><li>Optimizing data transfer with S3 Transfer Acceleration</li></ul>
|
|
<h2>Core Concepts</h2>
|
|
<h3>Concept 1: S3 Buckets and Storage Classes</h3>
|
|
<p><strong>S3 storage classes</strong>:</p>
|
|
<ul><li><strong>S3 Standard</strong>: Frequent access, low latency ($0.023/GB)</li><li><strong>S3 Intelligent-Tiering</strong>: Auto-moves between tiers</li><li><strong>S3 Standard-IA</strong>: Infrequent access ($0.0125/GB)</li><li><strong>S3 One Zone-IA</strong>: Single AZ, infrequent ($0.01/GB)</li><li><strong>S3 Glacier Instant</strong>: Archive with instant retrieval</li><li><strong>S3 Glacier Flexible</strong>: Archive with 1-5 min retrieval</li><li><strong>S3 Glacier Deep Archive</strong>: Cheapest, 12hr retrieval ($0.00099/GB)</li></ul>
|
|
<pre><code class="lang-python">import boto3
|
|
|
|
s3 = boto3.client('s3')
|
|
|
|
def create_bucket(bucket_name, region='us-east-1'):
|
|
"""Create S3 bucket with versioning and encryption"""
|
|
|
|
# Create bucket
|
|
if region == 'us-east-1':
|
|
s3.create_bucket(Bucket=bucket_name)
|
|
else:
|
|
s3.create_bucket(
|
|
Bucket=bucket_name,
|
|
CreateBucketConfiguration={'LocationConstraint': region}
|
|
)
|
|
|
|
# Enable versioning
|
|
s3.put_bucket_versioning(
|
|
Bucket=bucket_name,
|
|
VersioningConfiguration={'Status': 'Enabled'}
|
|
)
|
|
|
|
# Enable default encryption
|
|
s3.put_bucket_encryption(
|
|
Bucket=bucket_name,
|
|
ServerSideEncryptionConfiguration={
|
|
'Rules': [
|
|
{
|
|
'ApplyServerSideEncryptionByDefault': {
|
|
'SSEAlgorithm': 'AES256'
|
|
},
|
|
'BucketKeyEnabled': True
|
|
}
|
|
]
|
|
}
|
|
)
|
|
|
|
# Block public access
|
|
s3.put_public_access_block(
|
|
Bucket=bucket_name,
|
|
PublicAccessBlockConfiguration={
|
|
'BlockPublicAcls': True,
|
|
'IgnorePublicAcls': True,
|
|
'BlockPublicPolicy': True,
|
|
'RestrictPublicBuckets': True
|
|
}
|
|
)
|
|
|
|
print(f"Created bucket: {bucket_name}")
|
|
|
|
def upload_object(bucket, key, file_path, storage_class='STANDARD'):
|
|
"""Upload file to S3 with storage class"""
|
|
|
|
s3.upload_file(
|
|
file_path,
|
|
bucket,
|
|
key,
|
|
ExtraArgs={
|
|
'StorageClass': storage_class,
|
|
'ServerSideEncryption': 'AES256',
|
|
'Metadata': {
|
|
'uploaded-by': 'automation',
|
|
'environment': 'production'
|
|
}
|
|
}
|
|
)
|
|
|
|
print(f"Uploaded {key} to {bucket} ({storage_class})")</code></pre>
|
|
<h3>Concept 2: EBS Volumes</h3>
|
|
<p><strong>EBS volume types</strong>:</p>
|
|
<ul><li><strong>gp3</strong>: General purpose SSD (3,000-16,000 IOPS)</li><li><strong>io2</strong>: Provisioned IOPS SSD (64,000+ IOPS)</li><li><strong>st1</strong>: Throughput-optimized HDD (for big data)</li><li><strong>sc1</strong>: Cold HDD (lowest cost)</li></ul>
|
|
<pre><code class="lang-python">import boto3
|
|
|
|
ec2 = boto3.client('ec2')
|
|
|
|
def create_and_attach_volume(instance_id, size_gb=100, volume_type='gp3'):
|
|
"""Create EBS volume and attach to instance"""
|
|
|
|
# Get instance AZ
|
|
response = ec2.describe_instances(InstanceIds=[instance_id])
|
|
az = response['Reservations'][0]['Instances'][0]['Placement']['AvailabilityZone']
|
|
|
|
# Create volume
|
|
volume_response = ec2.create_volume(
|
|
AvailabilityZone=az,
|
|
Size=size_gb,
|
|
VolumeType=volume_type,
|
|
Iops=3000 if volume_type == 'gp3' else None, # gp3 baseline
|
|
Throughput=125 if volume_type == 'gp3' else None, # MB/s
|
|
Encrypted=True,
|
|
TagSpecifications=[
|
|
{
|
|
'ResourceType': 'volume',
|
|
'Tags': [
|
|
{'Key': 'Name', 'Value': f'data-volume-{instance_id}'},
|
|
{'Key': 'ManagedBy', 'Value': 'automation'}
|
|
]
|
|
}
|
|
]
|
|
)
|
|
|
|
volume_id = volume_response['VolumeId']
|
|
print(f"Created volume: {volume_id}")
|
|
|
|
# Wait for volume to be available
|
|
waiter = ec2.get_waiter('volume_available')
|
|
waiter.wait(VolumeIds=[volume_id])
|
|
|
|
# Attach volume
|
|
ec2.attach_volume(
|
|
Device='/dev/sdf',
|
|
InstanceId=instance_id,
|
|
VolumeId=volume_id
|
|
)
|
|
|
|
print(f"Attached {volume_id} to {instance_id}")
|
|
|
|
return volume_id
|
|
|
|
def create_snapshot(volume_id, description):
|
|
"""Create EBS snapshot for backup"""
|
|
|
|
response = ec2.create_snapshot(
|
|
VolumeId=volume_id,
|
|
Description=description,
|
|
TagSpecifications=[
|
|
{
|
|
'ResourceType': 'snapshot',
|
|
'Tags': [
|
|
{'Key': 'BackupType', 'Value': 'automated'},
|
|
{'Key': 'CreatedBy', 'Value': 'backup-lambda'}
|
|
]
|
|
}
|
|
]
|
|
)
|
|
|
|
snapshot_id = response['SnapshotId']
|
|
print(f"Creating snapshot: {snapshot_id}")
|
|
|
|
return snapshot_id</code></pre>
|
|
<h3>Concept 3: EFS (Elastic File System)</h3>
|
|
<p><strong>EFS use cases</strong>:</p>
|
|
<ul><li>Shared storage across multiple EC2 instances</li><li>Container persistent volumes</li><li>Content management systems</li><li>Development environments</li></ul>
|
|
<pre><code class="lang-python">import boto3
|
|
|
|
efs = boto3.client('efs')
|
|
|
|
def create_efs_file_system():
|
|
"""Create EFS file system with mount targets"""
|
|
|
|
# Create file system
|
|
response = efs.create_file_system(
|
|
PerformanceMode='generalPurpose', # or 'maxIO'
|
|
ThroughputMode='bursting', # or 'provisioned'
|
|
Encrypted=True,
|
|
Tags=[
|
|
{'Key': 'Name', 'Value': 'shared-storage'},
|
|
{'Key': 'Environment', 'Value': 'production'}
|
|
]
|
|
)
|
|
|
|
fs_id = response['FileSystemId']
|
|
print(f"Created EFS: {fs_id}")
|
|
|
|
# Wait for file system to be available
|
|
waiter = efs.get_waiter('file_system_available')
|
|
waiter.wait(FileSystemId=fs_id)
|
|
|
|
# Create mount targets in each subnet (AZ)
|
|
subnets = ['subnet-abc123', 'subnet-def456']
|
|
security_group = 'sg-0123456789abcdef0'
|
|
|
|
for subnet in subnets:
|
|
efs.create_mount_target(
|
|
FileSystemId=fs_id,
|
|
SubnetId=subnet,
|
|
SecurityGroups=[security_group]
|
|
)
|
|
|
|
print(f"Created mount targets for {fs_id}")
|
|
|
|
return fs_id
|
|
|
|
# Mount EFS on EC2 instance (user data script)
|
|
EFS_MOUNT_SCRIPT = """#!/bin/bash
|
|
# Install EFS utilities
|
|
yum install -y amazon-efs-utils
|
|
|
|
# Create mount point
|
|
mkdir -p /mnt/efs
|
|
|
|
# Mount EFS
|
|
mount -t efs -o tls {fs_id}:/ /mnt/efs
|
|
|
|
# Add to fstab for persistence
|
|
echo "{fs_id}:/ /mnt/efs efs defaults,_netdev,tls 0 0" >> /etc/fstab
|
|
"""</code></pre>
|
|
<hr>
|
|
<h2>Patterns</h2>
|
|
<h3>Pattern 1: S3 Lifecycle Policies</h3>
|
|
<p><strong>When to use</strong>: Automatic transition to cheaper storage classes</p>
|
|
<pre><code class="lang-python">def configure_lifecycle_policy(bucket_name):
|
|
"""Configure S3 lifecycle transitions and expiration"""
|
|
|
|
s3.put_bucket_lifecycle_configuration(
|
|
Bucket=bucket_name,
|
|
LifecycleConfiguration={
|
|
'Rules': [
|
|
{
|
|
'Id': 'archive-old-logs',
|
|
'Status': 'Enabled',
|
|
'Filter': {'Prefix': 'logs/'},
|
|
'Transitions': [
|
|
{
|
|
'Days': 30,
|
|
'StorageClass': 'STANDARD_IA'
|
|
},
|
|
{
|
|
'Days': 90,
|
|
'StorageClass': 'GLACIER_FLEXIBLE_RETRIEVAL'
|
|
},
|
|
{
|
|
'Days': 365,
|
|
'StorageClass': 'DEEP_ARCHIVE'
|
|
}
|
|
],
|
|
'Expiration': {
|
|
'Days': 2555 # 7 years
|
|
}
|
|
},
|
|
{
|
|
'Id': 'cleanup-temp-files',
|
|
'Status': 'Enabled',
|
|
'Filter': {'Prefix': 'temp/'},
|
|
'Expiration': {
|
|
'Days': 7
|
|
},
|
|
'AbortIncompleteMultipartUpload': {
|
|
'DaysAfterInitiation': 1
|
|
}
|
|
},
|
|
{
|
|
'Id': 'intelligent-tiering',
|
|
'Status': 'Enabled',
|
|
'Filter': {'Prefix': 'data/'},
|
|
'Transitions': [
|
|
{
|
|
'Days': 0,
|
|
'StorageClass': 'INTELLIGENT_TIERING'
|
|
}
|
|
]
|
|
}
|
|
]
|
|
}
|
|
)
|
|
|
|
print(f"Configured lifecycle policy for {bucket_name}")</code></pre>
|
|
<p><strong>Benefits</strong>:</p>
|
|
<ul><li>Automatic cost optimization</li><li>No manual intervention</li><li>Compliance retention policies</li></ul>
|
|
<h3>Pattern 2: S3 Event Notifications</h3>
|
|
<p><strong>Use case</strong>: Trigger Lambda on file upload</p>
|
|
<pre><code class="lang-python">def configure_s3_notifications(bucket_name, lambda_arn):
|
|
"""Configure S3 to trigger Lambda on object creation"""
|
|
|
|
# Grant S3 permission to invoke Lambda
|
|
lambda_client = boto3.client('lambda')
|
|
lambda_client.add_permission(
|
|
FunctionName=lambda_arn.split(':')[-1],
|
|
StatementId='s3-invoke-permission',
|
|
Action='lambda:InvokeFunction',
|
|
Principal='s3.amazonaws.com',
|
|
SourceArn=f'arn:aws:s3:::{bucket_name}'
|
|
)
|
|
|
|
# Configure notification
|
|
s3.put_bucket_notification_configuration(
|
|
Bucket=bucket_name,
|
|
NotificationConfiguration={
|
|
'LambdaFunctionConfigurations': [
|
|
{
|
|
'LambdaFunctionArn': lambda_arn,
|
|
'Events': ['s3:ObjectCreated:*'],
|
|
'Filter': {
|
|
'Key': {
|
|
'FilterRules': [
|
|
{'Name': 'prefix', 'Value': 'uploads/'},
|
|
{'Name': 'suffix', 'Value': '.jpg'}
|
|
]
|
|
}
|
|
}
|
|
}
|
|
]
|
|
}
|
|
)
|
|
|
|
print(f"Configured S3 notifications for {bucket_name}")</code></pre>
|
|
<h3>Pattern 3: Presigned URLs</h3>
|
|
<p><strong>Use case</strong>: Temporary access to private objects</p>
|
|
<pre><code class="lang-python">from botocore.exceptions import ClientError
|
|
|
|
def generate_presigned_url(bucket, key, expiration=3600):
|
|
"""Generate presigned URL for temporary access"""
|
|
|
|
try:
|
|
url = s3.generate_presigned_url(
|
|
'get_object',
|
|
Params={
|
|
'Bucket': bucket,
|
|
'Key': key
|
|
},
|
|
ExpiresIn=expiration # Seconds
|
|
)
|
|
|
|
return url
|
|
|
|
except ClientError as e:
|
|
print(f"Error generating URL: {e}")
|
|
return None
|
|
|
|
def generate_presigned_upload_url(bucket, key, expiration=3600):
|
|
"""Generate presigned URL for upload"""
|
|
|
|
url = s3.generate_presigned_url(
|
|
'put_object',
|
|
Params={
|
|
'Bucket': bucket,
|
|
'Key': key,
|
|
'ContentType': 'image/jpeg'
|
|
},
|
|
ExpiresIn=expiration,
|
|
HttpMethod='PUT'
|
|
)
|
|
|
|
return url
|
|
|
|
# Usage
|
|
download_url = generate_presigned_url('my-bucket', 'private/file.pdf', 300)
|
|
upload_url = generate_presigned_upload_url('my-bucket', 'uploads/new.jpg', 600)</code></pre>
|
|
<h3>Pattern 4: S3 Transfer Acceleration</h3>
|
|
<p><strong>Use case</strong>: Fast uploads from distant locations</p>
|
|
<pre><code class="lang-bash"># Enable Transfer Acceleration
|
|
aws s3api put-bucket-accelerate-configuration \
|
|
--bucket my-bucket \
|
|
--accelerate-configuration Status=Enabled</code></pre>
|
|
<pre><code class="lang-python">def upload_with_acceleration(bucket, key, file_path):
|
|
"""Upload using S3 Transfer Acceleration endpoint"""
|
|
|
|
# Create client with accelerate endpoint
|
|
s3_accelerate = boto3.client(
|
|
's3',
|
|
config=boto3.session.Config(
|
|
s3={'use_accelerate_endpoint': True}
|
|
)
|
|
)
|
|
|
|
# Upload file
|
|
s3_accelerate.upload_file(file_path, bucket, key)
|
|
|
|
print(f"Uploaded {key} using Transfer Acceleration")</code></pre>
|
|
<h3>Pattern 5: EBS Snapshot Automation</h3>
|
|
<p><strong>Use case</strong>: Automated backup strategy</p>
|
|
<pre><code class="lang-python">from datetime import datetime, timedelta
|
|
|
|
def backup_ebs_volumes(tag_key='Backup', tag_value='true'):
|
|
"""Create snapshots of tagged volumes"""
|
|
|
|
# Find volumes with backup tag
|
|
response = ec2.describe_volumes(
|
|
Filters=[
|
|
{'Name': f'tag:{tag_key}', 'Values': [tag_value]}
|
|
]
|
|
)
|
|
|
|
for volume in response['Volumes']:
|
|
volume_id = volume['VolumeId']
|
|
|
|
# Create snapshot
|
|
snapshot = ec2.create_snapshot(
|
|
VolumeId=volume_id,
|
|
Description=f'Automated backup {datetime.utcnow().isoformat()}',
|
|
TagSpecifications=[
|
|
{
|
|
'ResourceType': 'snapshot',
|
|
'Tags': [
|
|
{'Key': 'BackupDate', 'Value': datetime.utcnow().strftime('%Y-%m-%d')},
|
|
{'Key': 'VolumeId', 'Value': volume_id}
|
|
]
|
|
}
|
|
]
|
|
)
|
|
|
|
print(f"Created snapshot {snapshot['SnapshotId']} for {volume_id}")
|
|
|
|
def cleanup_old_snapshots(retention_days=30):
|
|
"""Delete snapshots older than retention period"""
|
|
|
|
cutoff_date = datetime.utcnow() - timedelta(days=retention_days)
|
|
|
|
# Find old snapshots
|
|
response = ec2.describe_snapshots(OwnerIds=['self'])
|
|
|
|
for snapshot in response['Snapshots']:
|
|
start_time = snapshot['StartTime'].replace(tzinfo=None)
|
|
|
|
if start_time < cutoff_date:
|
|
snapshot_id = snapshot['SnapshotId']
|
|
print(f"Deleting old snapshot: {snapshot_id}")
|
|
|
|
try:
|
|
ec2.delete_snapshot(SnapshotId=snapshot_id)
|
|
except ClientError as e:
|
|
print(f"Failed to delete {snapshot_id}: {e}")</code></pre>
|
|
<hr>
|
|
<h2>Quick Reference</h2>
|
|
<h3>Storage Service Selection</h3>
|
|
<p>| Use Case | Service | Type | Access Pattern | |----------|---------|------|----------------| | Objects, files, backups | S3 | Object | API/HTTP | | Block storage for EC2 | EBS | Block | Attached volume | | Shared file system | EFS | File | NFS mount | | Long-term archive | Glacier | Object | Rare retrieval |</p>
|
|
<h3>S3 Storage Class Costs (per GB/month)</h3>
|
|
<pre><code>Storage Class | Cost | Retrieval | Use Case
|
|
---------------------------|-----------|--------------|------------------
|
|
S3 Standard | $0.023 | Free | Active data
|
|
S3 Intelligent-Tiering | $0.023+ | Free | Unknown pattern
|
|
S3 Standard-IA | $0.0125 | $0.01/GB | Infrequent access
|
|
S3 Glacier Instant | $0.004 | $0.03/GB | Archive (instant)
|
|
S3 Glacier Flexible | $0.0036 | $0.01/GB | Archive (mins)
|
|
S3 Deep Archive | $0.00099 | $0.02/GB | Long-term (hours)</code></pre>
|
|
<h3>Key Guidelines</h3>
|
|
<pre><code>✅ DO: Enable versioning for critical data
|
|
✅ DO: Enable default encryption on all buckets
|
|
✅ DO: Use lifecycle policies to reduce costs
|
|
✅ DO: Block public access unless explicitly needed
|
|
✅ DO: Use presigned URLs for temporary access
|
|
✅ DO: Encrypt EBS volumes (especially production)
|
|
|
|
❌ DON'T: Make buckets public without justification
|
|
❌ DON'T: Store sensitive data without encryption
|
|
❌ DON'T: Use S3 Standard for infrequent access data
|
|
❌ DON'T: Forget to clean up old snapshots
|
|
❌ DON'T: Attach EBS volumes across AZs (not possible)</code></pre>
|
|
<hr>
|
|
<h2>Anti-Patterns</h2>
|
|
<h3>Critical Violations</h3>
|
|
<pre><code class="lang-python"># ❌ NEVER: Create public bucket without justification
|
|
s3.create_bucket(Bucket='my-bucket')
|
|
s3.put_bucket_acl(Bucket='my-bucket', ACL='public-read')
|
|
# All objects publicly accessible!
|
|
|
|
# ✅ CORRECT: Block public access by default
|
|
s3.put_public_access_block(
|
|
Bucket='my-bucket',
|
|
PublicAccessBlockConfiguration={
|
|
'BlockPublicAcls': True,
|
|
'IgnorePublicAcls': True,
|
|
'BlockPublicPolicy': True,
|
|
'RestrictPublicBuckets': True
|
|
}
|
|
)
|
|
# Use presigned URLs for temporary access</code></pre>
|
|
<p>❌ <strong>Public buckets</strong>: Data breaches, compliance violations, security incidents</p>
|
|
<p>✅ <strong>Correct approach</strong>: Block public access, use presigned URLs or CloudFront</p>
|
|
<h3>Common Mistakes</h3>
|
|
<pre><code class="lang-python"># ❌ Don't use S3 Standard for all data
|
|
s3.upload_file('archive.zip', 'my-bucket', 'data/archive.zip')
|
|
# Paying $0.023/GB for rarely accessed data
|
|
|
|
# ✅ Correct: Use appropriate storage class
|
|
s3.upload_file(
|
|
'archive.zip',
|
|
'my-bucket',
|
|
'data/archive.zip',
|
|
ExtraArgs={'StorageClass': 'GLACIER_FLEXIBLE_RETRIEVAL'}
|
|
)
|
|
# Paying $0.0036/GB for archival data</code></pre>
|
|
<p>❌ <strong>Wrong storage class</strong>: Overpaying for storage based on access patterns</p>
|
|
<p>✅ <strong>Better</strong>: Match storage class to access frequency, use lifecycle policies</p>
|
|
<hr>
|
|
<h2>Related Skills</h2>
|
|
<ul><li><code>aws-lambda-functions.md</code> - Process S3 events with Lambda</li><li><code>aws-ec2-compute.md</code> - Attach EBS volumes to EC2 instances</li><li><code>aws-databases.md</code> - Storage for database backups</li><li><code>aws-networking.md</code> - S3 access through VPC endpoints</li><li><code>aws-iam-security.md</code> - S3 bucket policies and IAM permissions</li></ul>
|
|
<hr>
|
|
<p><strong>Last Updated</strong>: 2025-10-25 <strong>Format Version</strong>: 1.0 (Atomic)</p>
|
|
<footer>
|
|
<div class="fmap">
|
|
<div><b>Build</b>
|
|
<a href="https://hanzo.app">hanzo.app</a> app builder
|
|
<a href="https://hanzo.codes">hanzo.codes</a> code with an agent
|
|
<a href="https://hanzoskills.com">hanzoskills.com</a> what an agent reads</div>
|
|
<div><b>Run</b>
|
|
<a href="https://console.hanzo.ai">console.hanzo.ai</a> cloud console
|
|
<a href="https://api.hanzo.ai">api.hanzo.ai</a> the API</div>
|
|
<div><b>Account</b>
|
|
<a href="https://hanzo.id">hanzo.id</a> sign in
|
|
<a href="https://pay.hanzo.ai">pay.hanzo.ai</a> top up
|
|
<a href="https://billing.hanzo.ai">billing.hanzo.ai</a> invoices and usage</div>
|
|
<div><b>Company</b>
|
|
<a href="https://hanzo.ai">hanzo.ai</a> canonical
|
|
<a href="https://hanzo.ventures">hanzo.ventures</a> ventures</div>
|
|
</div>
|
|
<div class="fend"><a href="/cloud-aws-storage.md">This page as plain text</a> · <a href="/llms.txt">Index for agents</a></div>
|
|
</footer>
|
|
</div></body></html> |