mirror of
https://github.com/hanzoai/skills.git
synced 2026-11-19 00:47:36 +00:00
hanzoskills.com/llms.txt now opens with the install line, the API reference index and the company index, so an agent that lands on any one of the three can find the rest. robots.txt claimed 537 documents while llms.txt counts 920. The number is gone from robots.txt; llms.txt derives it. public/ is regenerated from scripts/site.py, which it had fallen behind: the copy-prompt menu and agent bar were in the generator and not in the committed pages. tests/site.mjs passes against the result. Co-authored-by: Hanzo Dev <[email protected]>
553 lines
29 KiB
HTML
553 lines
29 KiB
HTML
<!doctype html>
|
|
<html lang="en"><head>
|
|
<meta charset="utf-8"><meta name="viewport" content="width=device-width,initial-scale=1">
|
|
<title>cloud-gcp-storage</title>
|
|
<meta name="color-scheme" content="dark">
|
|
<style>
|
|
/* Committed to one visual world: a dark instrument panel. No light variant —
|
|
the dot field and the hairline rules only read on black, and a naive
|
|
inversion would wash both out. Every colour is painted explicitly so the page
|
|
never borrows a host background. */
|
|
@font-face{font-family:'Zen';src:url('/fonts/Zen-Variable.woff2') format('woff2');font-weight:100 900;font-style:normal;font-display:swap}
|
|
@font-face{font-family:'Zen';src:url('/fonts/Zen-Italic-Variable.woff2') format('woff2');font-weight:100 900;font-style:italic;font-display:swap}
|
|
@font-face{font-family:'Zen Mono';src:url('/fonts/ZenMono-Variable.woff2') format('woff2');font-weight:100 900;font-style:normal;font-display:swap}
|
|
@font-face{font-family:'Zen Mono';src:url('/fonts/ZenMono-Italic-Variable.woff2') format('woff2');font-weight:100 900;font-style:italic;font-display:swap}
|
|
@font-face{font-family:'Zen Pixel Square';src:url('/fonts/ZenPixel-Square.woff2') format('woff2');font-style:normal;font-display:swap}
|
|
|
|
:root{
|
|
--ink:#f4f4f2; --soft:#9c9c98; --faint:#6a6a66; --ground:#000; --panel:#0a0a0a;
|
|
--line:#232323; --line-hi:#3a3a3a; --accent:#f4f4f2;
|
|
--font-zen-sans:'Zen',-apple-system,BlinkMacSystemFont,"Segoe UI",Helvetica,Arial,sans-serif;
|
|
--font-zen-mono:'Zen Mono',ui-monospace,"SF Mono",SFMono-Regular,Menlo,Consolas,monospace;
|
|
--font-zen-pixel-square:'Zen Pixel Square',monospace;
|
|
--mono:var(--font-zen-mono);
|
|
--sans:var(--font-zen-sans);
|
|
/* Controls are pills and surfaces are soft, as on hanzo.ai. */
|
|
--r:999px; --r-soft:14px;
|
|
}
|
|
*{box-sizing:border-box}
|
|
html{-webkit-text-size-adjust:100%}
|
|
body{margin:0;background:var(--ground);color:var(--ink);font-family:var(--sans);
|
|
font-size:16px;line-height:1.65;-webkit-font-smoothing:antialiased}
|
|
.wrap{max-width:60rem;margin:0 auto;padding:0 1.75rem}
|
|
|
|
/* The dot field. A halftone made of one repeating radial-gradient, faded out
|
|
along the diagonal by a mask so it reads as a wave rather than wallpaper. */
|
|
/* No overflow clip here: the dot field is inset to the box already, and a clip
|
|
would cut off the copy-prompt menu. */
|
|
.hero{position:relative;border-bottom:1px solid var(--line)}
|
|
.hero::before{content:"";position:absolute;inset:0;
|
|
background-image:radial-gradient(var(--line-hi) 1px, transparent 1px);
|
|
background-size:7px 7px;
|
|
-webkit-mask-image:linear-gradient(115deg,#000 0%,rgba(0,0,0,.55) 38%,transparent 72%);
|
|
mask-image:linear-gradient(115deg,#000 0%,rgba(0,0,0,.55) 38%,transparent 72%);
|
|
pointer-events:none}
|
|
.hero .wrap{position:relative;padding-top:5.5rem;padding-bottom:4rem}
|
|
|
|
nav{border-bottom:1px solid var(--line)}
|
|
nav .in{max-width:60rem;margin:0 auto;padding:1.1rem 1.75rem;display:flex;
|
|
justify-content:space-between;align-items:center;gap:1rem}
|
|
.brand{font-family:var(--mono);font-size:.85rem;letter-spacing:.14em;
|
|
text-transform:uppercase;color:var(--ink);text-decoration:none;
|
|
display:inline-flex;align-items:center;gap:.6rem}
|
|
.brand i{font-style:normal;color:var(--faint)}
|
|
.mark{width:1.15em;height:1.15em;display:block;flex:0 0 auto}
|
|
nav a.up{font-family:var(--mono);font-size:.72rem;letter-spacing:.14em;
|
|
text-transform:uppercase;color:var(--soft);text-decoration:none}
|
|
nav a.up:hover{color:var(--ink)}
|
|
|
|
h1{font-family:var(--mono);font-size:clamp(2rem,5.4vw,3.4rem);line-height:1.12;
|
|
letter-spacing:-.02em;margin:0 0 1.1rem;font-weight:500;text-wrap:balance}
|
|
h1 em{font-style:normal;color:var(--faint)}
|
|
.lede{color:var(--soft);font-size:1.05rem;max-width:52ch;margin:0 0 2.25rem}
|
|
|
|
.btns{display:flex;flex-wrap:wrap;gap:.7rem;margin:0 0 .5rem;align-items:center}
|
|
.btn{display:inline-flex;align-items:center;gap:.5rem;font-family:var(--mono);
|
|
font-size:.72rem;letter-spacing:.14em;text-transform:uppercase;
|
|
padding:.8rem 1.4rem;border:1px solid var(--line-hi);border-radius:var(--r);
|
|
color:var(--ink);text-decoration:none;background:transparent;cursor:pointer;
|
|
line-height:1.2;font-weight:400}
|
|
.btn:hover{background:#141414;text-decoration:none;border-color:var(--soft)}
|
|
.btn.solid{background:var(--ink);color:#000;border-color:var(--ink)}
|
|
.btn.solid:hover{background:#d8d8d4}
|
|
.btn .chev{width:.7em;height:.7em;flex:0 0 auto;transition:transform .15s ease}
|
|
.btn[aria-expanded="true"] .chev{transform:rotate(180deg)}
|
|
|
|
/* Copy-prompt selector. Each row copies the prompt for its own target. */
|
|
.copy{position:relative;display:inline-flex}
|
|
.menu{position:absolute;top:calc(100% + .5rem);left:0;z-index:20;min-width:21rem;
|
|
background:var(--panel);border:1px solid var(--line-hi);border-radius:var(--r-soft);
|
|
padding:.4rem;display:flex;flex-direction:column;
|
|
box-shadow:0 12px 32px rgba(0,0,0,.75)}
|
|
.menu[hidden]{display:none}
|
|
.mi{display:flex;align-items:center;gap:.7rem;width:100%;text-align:left;
|
|
padding:.62rem .8rem;border:0;border-radius:calc(var(--r-soft) - .35rem);
|
|
background:transparent;color:var(--ink);font-family:var(--mono);font-size:.76rem;
|
|
letter-spacing:.04em;text-decoration:none;cursor:pointer;line-height:1.3;
|
|
white-space:nowrap}
|
|
.mi:hover,.mi:focus-visible{background:#181818;outline:none}
|
|
.mi .mark{width:1.05rem;height:1.05rem;color:var(--soft)}
|
|
.mi:hover .mark{color:var(--ink)}
|
|
.mi u{margin-left:auto;color:var(--faint);text-decoration:none;font-size:.68rem;
|
|
letter-spacing:.1em;text-transform:uppercase}
|
|
|
|
/* Works-with bar: one scrolling row, name and mark per agent. */
|
|
.strip{border-bottom:1px solid var(--line);background:var(--panel)}
|
|
.strip .in{max-width:60rem;margin:0 auto;padding:.9rem 1.75rem;display:flex;
|
|
align-items:center;gap:1.25rem;font-family:var(--mono);font-size:.72rem;
|
|
letter-spacing:.1em;text-transform:uppercase;color:var(--faint)}
|
|
.strip b{color:var(--soft);font-weight:400;flex:0 0 auto;
|
|
border-right:1px solid var(--line);padding-right:1.25rem;align-self:stretch;
|
|
display:flex;align-items:center}
|
|
.bar{display:flex;align-items:center;gap:.5rem;flex:1 1 auto;min-width:0;
|
|
overflow-x:auto;scrollbar-width:none;scroll-snap-type:x proximity;
|
|
-webkit-overflow-scrolling:touch;
|
|
-webkit-mask-image:linear-gradient(90deg,#000 calc(100% - 3rem),transparent);
|
|
mask-image:linear-gradient(90deg,#000 calc(100% - 3rem),transparent)}
|
|
.bar::-webkit-scrollbar{display:none}
|
|
.agent{display:inline-flex;align-items:center;gap:.45rem;flex:0 0 auto;
|
|
padding:.4rem .85rem;border:1px solid var(--line);border-radius:var(--r);
|
|
color:var(--soft);white-space:nowrap;scroll-snap-align:start;
|
|
text-decoration:none;background:var(--ground)}
|
|
.agent:hover{color:var(--ink);border-color:var(--line-hi);text-decoration:none}
|
|
.agent .mark{width:.95rem;height:.95rem}
|
|
|
|
h2{font-family:var(--mono);font-size:1.1rem;letter-spacing:-.01em;margin:2.75rem 0 .75rem;font-weight:500}
|
|
h3{font-size:1rem;margin:2rem 0 .5rem;font-weight:600}
|
|
h4,h5,h6{font-size:.94rem;margin:1.5rem 0 .5rem;font-weight:600}
|
|
p{margin:0 0 1rem;max-width:70ch}
|
|
ul,ol{margin:0 0 1rem;padding-left:1.35rem;max-width:70ch}
|
|
li{margin:.22rem 0}
|
|
a{color:var(--ink);text-decoration:underline;text-underline-offset:3px;
|
|
text-decoration-color:var(--line-hi)}
|
|
a:hover{text-decoration-color:var(--ink)}
|
|
a:focus-visible{outline:1px solid var(--ink);outline-offset:3px}
|
|
code{font-family:var(--mono);font-size:.85em;background:var(--panel);
|
|
border:1px solid var(--line);border-radius:5px;padding:.1em .36em}
|
|
pre{background:var(--panel);border:1px solid var(--line);padding:1rem 1.1rem;
|
|
border-radius:var(--r-soft);overflow-x:auto;margin:0 0 1.25rem}
|
|
pre code{background:none;border:0;padding:0;font-size:.82rem;line-height:1.7}
|
|
blockquote{margin:0 0 1rem;padding-left:1rem;border-left:1px solid var(--line-hi);color:var(--soft)}
|
|
hr{border:0;border-top:1px solid var(--line);margin:2.5rem 0}
|
|
table{border-collapse:collapse;width:100%;margin:0 0 1.25rem;font-size:.92rem;
|
|
display:block;overflow-x:auto}
|
|
th,td{border:1px solid var(--line);padding:.5rem .7rem;text-align:left}
|
|
|
|
.curl{display:block;background:var(--panel);border:1px solid var(--line);
|
|
border-radius:var(--r-soft);padding:.95rem 1.1rem;font-family:var(--mono);
|
|
font-size:.8rem;color:var(--soft);overflow-x:auto;margin:0 0 2.5rem;
|
|
white-space:pre;line-height:1.9}
|
|
.curl b{color:var(--ink);font-weight:400}
|
|
.curl i{font-style:normal;color:var(--faint)}
|
|
|
|
/* index */
|
|
#q{width:100%;padding:.85rem 1.2rem;font-family:var(--mono);font-size:.85rem;color:var(--ink);
|
|
background:var(--panel);border:1px solid var(--line);border-radius:var(--r);margin:0 0 .6rem}
|
|
#q::placeholder{color:var(--faint);text-transform:uppercase;letter-spacing:.1em;font-size:.72rem}
|
|
#q:focus{outline:none;border-color:var(--soft)}
|
|
#count{color:var(--faint);font-family:var(--mono);font-size:.72rem;letter-spacing:.1em;
|
|
text-transform:uppercase;margin:0 0 2rem}
|
|
.cat{margin:2.5rem 0 .5rem;font-family:var(--mono);font-size:.7rem;letter-spacing:.16em;
|
|
text-transform:uppercase;color:var(--faint);border-bottom:1px solid var(--line);
|
|
padding-bottom:.45rem}
|
|
.row{display:block;padding:.6rem .9rem;margin:0 -.9rem;border-bottom:1px solid var(--line);
|
|
border-radius:var(--r-soft);text-decoration:none}
|
|
.row:hover{background:#0a0a0a}
|
|
.nm{font-family:var(--mono);font-size:.84rem;color:var(--ink);display:block}
|
|
.ds{color:var(--soft);font-size:.9rem;display:block;margin-top:.12rem}
|
|
footer{margin-top:4rem;padding:2rem 0 4rem;border-top:1px solid var(--line);
|
|
color:var(--faint);font-size:.85rem}
|
|
footer a{color:var(--soft)}
|
|
.fmap{display:grid;gap:1.75rem;grid-template-columns:repeat(auto-fit,minmax(11rem,1fr));
|
|
margin:0 0 2rem;font-size:.85rem}
|
|
.fmap b{display:block;font-family:var(--mono);font-size:.68rem;letter-spacing:.16em;
|
|
text-transform:uppercase;color:var(--faint);font-weight:400;margin:0 0 .6rem;
|
|
border-bottom:1px solid var(--line);padding-bottom:.4rem}
|
|
.fmap a{display:inline-block;color:var(--ink);text-decoration:none;margin:.3rem .35rem 0 0}
|
|
.fmap a:hover{text-decoration:underline}
|
|
.fmap div{color:var(--faint);line-height:1.5}
|
|
.fend{border-top:1px solid var(--line);padding-top:1.25rem;font-size:.82rem}
|
|
|
|
@media (max-width:640px){
|
|
.hero .wrap{padding-top:3.25rem;padding-bottom:2.75rem}
|
|
.btns{gap:.5rem}
|
|
.btn{padding:.75rem 1.15rem;font-size:.68rem}
|
|
.copy,.copy .btn{width:100%;justify-content:center}
|
|
.menu{left:0;right:0;min-width:0}
|
|
.strip .in{flex-direction:column;align-items:flex-start;gap:.6rem;padding:.9rem 1.75rem}
|
|
.strip b{border-right:0;padding-right:0}
|
|
.bar{width:100%}
|
|
}
|
|
</style>
|
|
<link rel="alternate" type="text/markdown" href="/cloud-gcp-storage.md">
|
|
|
|
<link rel="icon" href="/favicon.svg">
|
|
<link rel="canonical" href="https://hanzoskills.com/cloud-gcp-storage">
|
|
</head>
|
|
<!-- Reading this with a machine? Every page here is also plain markdown.
|
|
This one: https://hanzoskills.com/cloud-gcp-storage.md
|
|
Index: https://hanzoskills.com/llms.txt
|
|
Corpus: https://hanzoskills.com/all.md
|
|
Append .md to any path for the markdown twin. -->
|
|
<body>
|
|
<nav><div class="in">
|
|
<a class="brand" href="/"><svg class="mark" viewBox="0 0 67 67" fill="currentColor" aria-hidden="true"><path d="M22.21 67V44.6369H0V67H22.21Z"/><path d="M66.7038 22.3184H22.2534L0.0878906 44.6367H44.4634L66.7038 22.3184Z"/><path d="M22.21 0H0V22.3184H22.21V0Z"/><path d="M66.7198 0H44.5098V22.3184H66.7198V0Z"/><path d="M66.7198 67V44.6369H44.5098V67H66.7198Z"/></svg> HANZO<i>SKILLS</i></a>
|
|
<a class="up" href="/">All skills</a>
|
|
</div></nav>
|
|
|
|
<div class="wrap">
|
|
<h1>cloud-gcp-storage</h1><p class="lede">Google Cloud storage services including Cloud Storage, Persistent Disk, and Filestore</p><h1>GCP Storage Services</h1>
|
|
<p><strong>Scope</strong>: Cloud Storage buckets, Persistent Disk, Filestore, and data transfer tools <strong>Lines</strong>: ~300 <strong>Last Updated</strong>: 2025-10-25 <strong>Format Version</strong>: 1.0 (Atomic)</p>
|
|
<hr>
|
|
<h2>When to Use This Skill</h2>
|
|
<p>Activate this skill when:</p>
|
|
<ul><li>Storing objects in Cloud Storage buckets with different storage classes</li><li>Attaching persistent disks to Compute Engine instances</li><li>Setting up shared file storage with Filestore (NFS)</li><li>Implementing object lifecycle management and retention policies</li><li>Transferring large datasets to Google Cloud</li><li>Configuring versioning, encryption, and access controls for storage</li><li>Optimizing storage costs with appropriate storage classes</li><li>Generating signed URLs for temporary access to private objects</li></ul>
|
|
<h2>Core Concepts</h2>
|
|
<h3>Concept 1: Cloud Storage Classes</h3>
|
|
<p><strong>Storage tiers</strong>:</p>
|
|
<ul><li><strong>Standard</strong>: Hot data, frequent access, no minimum storage duration</li><li><strong>Nearline</strong>: Accessed <1/month, 30-day minimum, backups, disaster recovery</li><li><strong>Coldline</strong>: Accessed <1/quarter, 90-day minimum, archival data</li><li><strong>Archive</strong>: Accessed <1/year, 365-day minimum, long-term compliance</li></ul>
|
|
<pre><code class="lang-bash"># Create bucket with Standard storage class
|
|
gsutil mb -c STANDARD -l us-central1 gs://my-app-data
|
|
|
|
# Create bucket with Nearline for backups
|
|
gsutil mb -c NEARLINE -l us-central1 gs://my-backups
|
|
|
|
# Upload file to specific storage class
|
|
gsutil -h "x-goog-storage-class:COLDLINE" cp archive.tar.gz gs://my-archives/
|
|
|
|
# Change storage class of existing object
|
|
gsutil rewrite -s ARCHIVE gs://my-backups/old-backup.tar.gz</code></pre>
|
|
<h3>Concept 2: Persistent Disk Types</h3>
|
|
<p><strong>Disk options</strong>:</p>
|
|
<ul><li><strong>pd-standard</strong> (HDD): Bulk storage, sequential I/O, lowest cost</li><li><strong>pd-balanced</strong> (SSD): Balanced price/performance, most workloads</li><li><strong>pd-ssd</strong>: High IOPS, low latency, databases</li><li><strong>pd-extreme</strong>: Highest performance, customizable IOPS</li></ul>
|
|
<pre><code class="lang-python">from google.cloud import compute_v1
|
|
|
|
def create_instance_with_disk(project_id, zone, instance_name):
|
|
client = compute_v1.InstancesClient()
|
|
|
|
# Create instance with balanced persistent disk
|
|
instance = compute_v1.Instance(
|
|
name=instance_name,
|
|
machine_type=f"zones/{zone}/machineTypes/n2-standard-4",
|
|
disks=[
|
|
# Boot disk (pd-balanced, 20GB)
|
|
compute_v1.AttachedDisk(
|
|
auto_delete=True,
|
|
boot=True,
|
|
initialize_params=compute_v1.AttachedDiskInitializeParams(
|
|
source_image="projects/debian-cloud/global/images/family/debian-11",
|
|
disk_size_gb=20,
|
|
disk_type=f"zones/{zone}/diskTypes/pd-balanced"
|
|
)
|
|
),
|
|
# Data disk (pd-ssd, 500GB)
|
|
compute_v1.AttachedDisk(
|
|
auto_delete=False, # Preserve disk after instance deletion
|
|
initialize_params=compute_v1.AttachedDiskInitializeParams(
|
|
disk_size_gb=500,
|
|
disk_type=f"zones/{zone}/diskTypes/pd-ssd"
|
|
)
|
|
)
|
|
],
|
|
network_interfaces=[
|
|
compute_v1.NetworkInterface(
|
|
network="global/networks/default"
|
|
)
|
|
]
|
|
)
|
|
|
|
operation = client.insert(project=project_id, zone=zone, instance_resource=instance)
|
|
return operation.result()</code></pre>
|
|
<h3>Concept 3: Object Lifecycle Management</h3>
|
|
<p><strong>Lifecycle actions</strong>:</p>
|
|
<ul><li><strong>SetStorageClass</strong>: Transition to cheaper storage class after time</li><li><strong>Delete</strong>: Remove objects after expiration</li><li>Conditions: age, created_before, number_of_newer_versions</li></ul>
|
|
<pre><code class="lang-json">{
|
|
"lifecycle": {
|
|
"rule": [
|
|
{
|
|
"action": {"type": "SetStorageClass", "storageClass": "NEARLINE"},
|
|
"condition": {"age": 30}
|
|
},
|
|
{
|
|
"action": {"type": "SetStorageClass", "storageClass": "COLDLINE"},
|
|
"condition": {"age": 90}
|
|
},
|
|
{
|
|
"action": {"type": "Delete"},
|
|
"condition": {"age": 365}
|
|
}
|
|
]
|
|
}
|
|
}</code></pre>
|
|
<pre><code class="lang-bash"># Apply lifecycle policy to bucket
|
|
gsutil lifecycle set lifecycle.json gs://my-bucket</code></pre>
|
|
<hr>
|
|
<h2>Patterns</h2>
|
|
<h3>Pattern 1: Versioning with Retention Policy</h3>
|
|
<p><strong>When to use</strong>:</p>
|
|
<ul><li>Protect against accidental deletion or overwrite</li><li>Meet compliance requirements for data retention</li></ul>
|
|
<pre><code class="lang-bash"># ❌ Bad: No versioning, data can be permanently lost
|
|
gsutil mb gs://critical-data
|
|
# Accidental overwrite loses data permanently
|
|
|
|
# ✅ Good: Enable versioning with retention policy
|
|
gsutil mb gs://critical-data
|
|
gsutil versioning set on gs://critical-data
|
|
|
|
# Set retention policy (data cannot be deleted for 7 years)
|
|
gsutil retention set 7y gs://critical-data
|
|
|
|
# Lock retention policy (irreversible - use carefully!)
|
|
gsutil retention lock gs://critical-data</code></pre>
|
|
<p><strong>Benefits</strong>:</p>
|
|
<ul><li>Recover from accidental deletion or corruption</li><li>Compliance with regulations (HIPAA, FINRA, etc.)</li><li>Immutable storage for audit logs</li></ul>
|
|
<h3>Pattern 2: Regional Persistent Disk for HA</h3>
|
|
<p><strong>Use case</strong>: Database high availability across zones in a region</p>
|
|
<pre><code class="lang-bash"># Create regional persistent disk (replicated across 2 zones)
|
|
gcloud compute disks create db-disk \
|
|
--size=500GB \
|
|
--type=pd-ssd \
|
|
--region=us-central1 \
|
|
--replica-zones=us-central1-a,us-central1-b
|
|
|
|
# Attach to instance in zone A
|
|
gcloud compute instances attach-disk db-primary \
|
|
--disk=db-disk \
|
|
--zone=us-central1-a
|
|
|
|
# If instance fails, disk can be attached to instance in zone B
|
|
gcloud compute instances detach-disk db-primary \
|
|
--disk=db-disk \
|
|
--zone=us-central1-a
|
|
|
|
gcloud compute instances attach-disk db-secondary \
|
|
--disk=db-disk \
|
|
--zone=us-central1-b</code></pre>
|
|
<h3>Pattern 3: Signed URLs for Temporary Access</h3>
|
|
<p><strong>Use case</strong>: Grant time-limited access to private objects without IAM changes</p>
|
|
<pre><code class="lang-python">from google.cloud import storage
|
|
from datetime import timedelta
|
|
|
|
def generate_signed_url(bucket_name, blob_name, expiration_minutes=15):
|
|
storage_client = storage.Client()
|
|
bucket = storage_client.bucket(bucket_name)
|
|
blob = bucket.blob(blob_name)
|
|
|
|
# Generate signed URL valid for 15 minutes
|
|
url = blob.generate_signed_url(
|
|
version="v4",
|
|
expiration=timedelta(minutes=expiration_minutes),
|
|
method="GET"
|
|
)
|
|
|
|
return url
|
|
|
|
# Example usage
|
|
url = generate_signed_url("private-reports", "report-2025-10.pdf")
|
|
# Share URL with user - valid for 15 minutes, no authentication needed</code></pre>
|
|
<h3>Pattern 4: Resumable Uploads for Large Files</h3>
|
|
<p><strong>Use case</strong>: Upload large files with automatic retry on network interruption</p>
|
|
<pre><code class="lang-python">from google.cloud import storage
|
|
|
|
def resumable_upload(bucket_name, source_file, destination_blob):
|
|
storage_client = storage.Client()
|
|
bucket = storage_client.bucket(bucket_name)
|
|
blob = bucket.blob(destination_blob)
|
|
|
|
# Enable resumable upload (automatic for files >5MB)
|
|
# If upload fails, it automatically resumes from where it left off
|
|
blob.upload_from_filename(
|
|
source_file,
|
|
timeout=3600 # 1 hour timeout for large files
|
|
)
|
|
|
|
print(f"Uploaded {source_file} to gs://{bucket_name}/{destination_blob}")
|
|
|
|
# Upload 10GB file - automatically handles interruptions
|
|
resumable_upload("my-bucket", "/data/large-dataset.tar.gz", "datasets/2025-10.tar.gz")</code></pre>
|
|
<h3>Pattern 5: Filestore for Shared NFS Storage</h3>
|
|
<p><strong>Use case</strong>: Multiple VMs need access to shared file system</p>
|
|
<pre><code class="lang-bash"># Create Filestore instance (1TB, BASIC_HDD tier)
|
|
gcloud filestore instances create shared-storage \
|
|
--location=us-central1-a \
|
|
--tier=BASIC_HDD \
|
|
--file-share=name=data,capacity=1TB \
|
|
--network=name=default
|
|
|
|
# Mount on Compute Engine instance
|
|
sudo apt-get install nfs-common
|
|
sudo mkdir -p /mnt/shared
|
|
sudo mount 10.0.0.2:/data /mnt/shared # Use IP from filestore instance
|
|
|
|
# Add to /etc/fstab for automatic mounting
|
|
echo "10.0.0.2:/data /mnt/shared nfs defaults 0 0" | sudo tee -a /etc/fstab</code></pre>
|
|
<h3>Pattern 6: Storage Transfer Service</h3>
|
|
<p><strong>Use case</strong>: Migrate large datasets from S3, Azure, or on-premises to Cloud Storage</p>
|
|
<pre><code class="lang-bash"># Create transfer job from AWS S3 to Cloud Storage
|
|
gcloud transfer jobs create s3://source-bucket gs://dest-bucket \
|
|
--source-creds-file=aws-creds.json \
|
|
--description="Migrate production data" \
|
|
--schedule-starts=2025-10-25T00:00:00Z \
|
|
--schedule-repeats-every=24h \
|
|
--delete-from-source-after-transfer # Optional: clean up source
|
|
|
|
# Monitor transfer job
|
|
gcloud transfer operations list --job-names=projects/PROJECT/transferJobs/JOB</code></pre>
|
|
<h3>Pattern 7: Customer-Managed Encryption Keys (CMEK)</h3>
|
|
<p><strong>Use case</strong>: Control encryption keys for compliance requirements</p>
|
|
<pre><code class="lang-bash"># Create Cloud KMS key ring and key
|
|
gcloud kms keyrings create my-keyring --location=us-central1
|
|
gcloud kms keys create storage-key \
|
|
--keyring=my-keyring \
|
|
--location=us-central1 \
|
|
--purpose=encryption
|
|
|
|
# Create bucket with CMEK
|
|
gsutil mb \
|
|
-p my-project \
|
|
-c STANDARD \
|
|
-l us-central1 \
|
|
-k projects/my-project/locations/us-central1/keyRings/my-keyring/cryptoKeys/storage-key \
|
|
gs://encrypted-bucket
|
|
|
|
# Upload object with CMEK
|
|
gsutil -o "GSUtil:encryption_key=..." cp file.txt gs://encrypted-bucket/</code></pre>
|
|
<h3>Pattern 8: Disk Snapshots for Backup</h3>
|
|
<p><strong>Use case</strong>: Create point-in-time backups of persistent disks</p>
|
|
<pre><code class="lang-bash"># Create snapshot of persistent disk
|
|
gcloud compute disks snapshot data-disk \
|
|
--snapshot-names=data-disk-2025-10-25 \
|
|
--zone=us-central1-a \
|
|
--storage-location=us-central1
|
|
|
|
# Create snapshot schedule for automatic backups
|
|
gcloud compute resource-policies create snapshot-schedule daily-backup \
|
|
--region=us-central1 \
|
|
--max-retention-days=7 \
|
|
--on-source-disk-delete=keep-auto-snapshots \
|
|
--daily-schedule \
|
|
--start-time=02:00
|
|
|
|
# Attach schedule to disk
|
|
gcloud compute disks add-resource-policies data-disk \
|
|
--resource-policies=daily-backup \
|
|
--zone=us-central1-a
|
|
|
|
# Restore from snapshot
|
|
gcloud compute disks create restored-disk \
|
|
--source-snapshot=data-disk-2025-10-25 \
|
|
--zone=us-central1-a</code></pre>
|
|
<hr>
|
|
<h2>Quick Reference</h2>
|
|
<h3>Storage Service Comparison</h3>
|
|
<pre><code>Service | Type | Use Case | Access | Cost
|
|
----------------|-----------|---------------------------|--------------|-------
|
|
Cloud Storage | Object | Unstructured data, media | HTTP/gsutil | $0.020-0.012/GB/mo
|
|
Persistent Disk | Block | VM attached storage | VM only | $0.040-0.170/GB/mo
|
|
Filestore | File/NFS | Shared filesystem | NFS mount | $0.20-0.30/GB/mo</code></pre>
|
|
<h3>Storage Class Pricing (Standard region)</h3>
|
|
<pre><code>Class | Storage $/GB/mo | Retrieval $/GB | Minimum Duration
|
|
----------|-----------------|----------------|------------------
|
|
Standard | $0.020 | Free | None
|
|
Nearline | $0.010 | $0.01 | 30 days
|
|
Coldline | $0.004 | $0.02 | 90 days
|
|
Archive | $0.0012 | $0.05 | 365 days</code></pre>
|
|
<h3>Key gsutil Commands</h3>
|
|
<pre><code class="lang-bash"># Bucket operations
|
|
gsutil mb -c STANDARD -l LOCATION gs://BUCKET
|
|
gsutil ls gs://BUCKET
|
|
gsutil du -s gs://BUCKET # Check size
|
|
gsutil rm -r gs://BUCKET
|
|
|
|
# Object operations
|
|
gsutil cp LOCAL gs://BUCKET/
|
|
gsutil cp gs://BUCKET/file .
|
|
gsutil mv gs://BUCKET/old gs://BUCKET/new
|
|
gsutil rm gs://BUCKET/file
|
|
|
|
# Parallel uploads (faster for many files)
|
|
gsutil -m cp -r /local/dir gs://BUCKET/
|
|
|
|
# Set metadata
|
|
gsutil setmeta -h "Cache-Control:public,max-age=3600" gs://BUCKET/file</code></pre>
|
|
<h3>Key Guidelines</h3>
|
|
<pre><code>✅ DO: Use lifecycle policies to transition to cheaper storage classes
|
|
✅ DO: Enable versioning for critical data
|
|
✅ DO: Use regional persistent disks for database high availability
|
|
✅ DO: Implement retention policies for compliance requirements
|
|
✅ DO: Use signed URLs instead of making objects public
|
|
✅ DO: Choose appropriate storage class based on access frequency
|
|
|
|
❌ DON'T: Store frequently accessed data in Archive class (expensive retrieval)
|
|
❌ DON'T: Use Filestore for object storage (use Cloud Storage instead)
|
|
❌ DON'T: Attach persistent disks to multiple VMs in read-write mode (data corruption)
|
|
❌ DON'T: Delete snapshots needed for disaster recovery
|
|
❌ DON'T: Use pd-extreme unless you've confirmed IOPS requirements</code></pre>
|
|
<hr>
|
|
<h2>Anti-Patterns</h2>
|
|
<h3>Critical Violations</h3>
|
|
<pre><code class="lang-bash"># ❌ NEVER: Lock retention policy without careful consideration
|
|
gsutil retention lock gs://bucket # IRREVERSIBLE - cannot be unlocked!
|
|
|
|
# ✅ CORRECT: Test retention policy first, then lock only if required by compliance
|
|
gsutil retention set 7y gs://bucket
|
|
# Verify policy works as expected for several months
|
|
# Only lock if legally required:
|
|
# gsutil retention lock gs://bucket</code></pre>
|
|
<p>❌ <strong>Premature retention lock</strong>: Locking a retention policy is permanent. You cannot delete the bucket or objects until retention period expires, even if you delete the project. ✅ <strong>Correct approach</strong>: Test retention policies thoroughly before locking. Only lock if required by compliance regulations.</p>
|
|
<h3>Common Mistakes</h3>
|
|
<pre><code class="lang-python"># ❌ Don't: Use Archive class for frequently accessed data
|
|
bucket = storage_client.bucket("user-uploads")
|
|
blob = bucket.blob("profile.jpg")
|
|
blob.upload_from_filename("profile.jpg")
|
|
blob.update_storage_class("ARCHIVE") # $0.05/GB retrieval cost!
|
|
|
|
# ✅ Correct: Use Standard class for hot data, lifecycle policy for cold data
|
|
blob.upload_from_filename("profile.jpg")
|
|
# Let lifecycle policy transition to cheaper class after 90 days</code></pre>
|
|
<p>❌ <strong>Wrong storage class</strong>: Using Archive class for frequently accessed data results in high retrieval costs that exceed storage savings. ✅ <strong>Better</strong>: Use Standard for hot data, Nearline for warm, Coldline/Archive for cold. Use lifecycle policies to automate transitions.</p>
|
|
<pre><code class="lang-bash"># ❌ Don't: Attach persistent disk to multiple instances in read-write mode
|
|
gcloud compute instances attach-disk vm-1 --disk=shared-disk --mode=rw
|
|
gcloud compute instances attach-disk vm-2 --disk=shared-disk --mode=rw
|
|
# Results in data corruption!
|
|
|
|
# ✅ Correct: Use Filestore for shared read-write access
|
|
gcloud filestore instances create shared-storage \
|
|
--location=us-central1-a \
|
|
--tier=BASIC_HDD \
|
|
--file-share=name=data,capacity=1TB \
|
|
--network=name=default</code></pre>
|
|
<p>❌ <strong>Multi-attach read-write</strong>: Attaching a persistent disk to multiple VMs in read-write mode causes data corruption. ✅ <strong>Better</strong>: Use Filestore (NFS) for shared file access, or attach disk in read-only mode to additional VMs.</p>
|
|
<pre><code class="lang-bash"># ❌ Don't: Delete original snapshots when using incremental snapshots
|
|
gcloud compute snapshots delete snapshot-1 # Breaks snapshot chain!
|
|
|
|
# ✅ Correct: Keep base snapshot or understand incremental chain
|
|
# Snapshots are incremental - deleting old snapshots is safe
|
|
# GCP automatically maintains snapshot chain integrity
|
|
gcloud compute snapshots delete snapshot-1 # Actually safe - GCP handles dependencies</code></pre>
|
|
<p>❌ <strong>Snapshot chain confusion</strong>: GCP snapshots are incremental but deleting old snapshots is safe because GCP automatically consolidates data. ✅ <strong>Better</strong>: Understand that GCP manages snapshot dependencies. Use snapshot schedules with retention policies to automate cleanup.</p>
|
|
<hr>
|
|
<h2>Related Skills</h2>
|
|
<ul><li><code>gcp-compute.md</code> - Attaching persistent disks to Compute Engine instances</li><li><code>gcp-iam-security.md</code> - Bucket IAM policies and service account access</li><li><code>gcp-databases.md</code> - Persistent disk configuration for database workloads</li><li><code>gcp-networking.md</code> - Private Google Access for storage without public IPs</li></ul>
|
|
<hr>
|
|
<p><strong>Last Updated</strong>: 2025-10-25 <strong>Format Version</strong>: 1.0 (Atomic)</p>
|
|
<footer>
|
|
<div class="fmap">
|
|
<div><b>Build</b>
|
|
<a href="https://hanzo.app">hanzo.app</a> app builder
|
|
<a href="https://hanzo.codes">hanzo.codes</a> code with an agent
|
|
<a href="https://hanzoskills.com">hanzoskills.com</a> what an agent reads</div>
|
|
<div><b>Run</b>
|
|
<a href="https://console.hanzo.ai">console.hanzo.ai</a> cloud console
|
|
<a href="https://api.hanzo.ai">api.hanzo.ai</a> the API</div>
|
|
<div><b>Account</b>
|
|
<a href="https://hanzo.id">hanzo.id</a> sign in
|
|
<a href="https://pay.hanzo.ai">pay.hanzo.ai</a> top up
|
|
<a href="https://billing.hanzo.ai">billing.hanzo.ai</a> invoices and usage</div>
|
|
<div><b>Company</b>
|
|
<a href="https://hanzo.ai">hanzo.ai</a> canonical
|
|
<a href="https://hanzo.ventures">hanzo.ventures</a> ventures</div>
|
|
</div>
|
|
<div class="fend"><a href="/cloud-gcp-storage.md">This page as plain text</a> · <a href="/llms.txt">Index for agents</a></div>
|
|
</footer>
|
|
</div></body></html> |