AWS S3 Basics Explained — Buckets, Objects, and Real-World Storage Patterns

#!/bin/bash
# ── Prerequisites: AWS CLI installed and configured with `aws configure` ──

# 1. Create a new bucket in us-east-1
#    Bucket names: lowercase, 3-63 chars, no underscores, globally unique
aws s3api create-bucket \
  --bucket acme-corp-user-uploads-dev \
  --region us-east-1
# Note: us-east-1 is the only region that does NOT need a LocationConstraint.
# Every other region requires --create-bucket-configuration LocationConstraint=<region>

# 2. Upload a local file as an S3 object
#    The key here is 'avatars/user-4821/profile.jpg'
#    S3 doesn't create a folder — 'avatars/user-4821/' is just part of the key name
aws s3 cp ./profile.jpg \
  s3://acme-corp-user-uploads-dev/avatars/user-4821/profile.jpg

# 3. List objects using a prefix filter (simulates folder browsing)
#    This returns ONLY objects whose key starts with 'avatars/user-4821/'
aws s3 ls s3://acme-corp-user-uploads-dev/avatars/user-4821/

# 4. Download the object back to verify the round-trip
aws s3 cp \
  s3://acme-corp-user-uploads-dev/avatars/user-4821/profile.jpg \
  ./profile_downloaded.jpg

# 5. Delete the object
aws s3 rm s3://acme-corp-user-uploads-dev/avatars/user-4821/profile.jpg

import boto3
import json
from datetime import datetime

# ── Boto3 picks up credentials from env vars, ~/.aws/credentials, or IAM role ──
s3_client = boto3.client('s3', region_name='us-east-1')

BUCKET_NAME = 'acme-corp-user-uploads-dev'

# ── PART 1: Attach a bucket policy ──
# This policy allows ONLY our application's IAM role to read objects
# under the 'invoices/' prefix. Nothing else in the bucket is affected.
bucket_policy = {
    "Version": "2012-10-17",
    "Statement": [
        {
            "Sid": "AllowAppRoleInvoiceRead",
            "Effect": "Allow",
            "Principal": {
                # Replace with your actual IAM role ARN
                "AWS": "arn:aws:iam::123456789012:role/AcmeAppServerRole"
            },
            "Action": "s3:GetObject",
            # The /* at the end means: any object under this prefix
            "Resource": f"arn:aws:s3:::{BUCKET_NAME}/invoices/*"
        }
    ]
}

s3_client.put_bucket_policy(
    Bucket=BUCKET_NAME,
    Policy=json.dumps(bucket_policy)  # Policy must be serialised to a JSON string
)
print(f"Bucket policy applied to {BUCKET_NAME}")

# ── PART 2: Generate a presigned URL for secure, time-limited object access ──
# Use case: a user clicks 'Download Invoice' in your web app.
# Your backend generates this URL and returns it. The browser hits S3 directly.
# The object itself stays private the whole time.

object_key = 'invoices/2024/jan/invoice-001.pdf'
expiry_seconds = 3600  # URL valid for exactly 1 hour

presigned_download_url = s3_client.generate_presigned_url(
    ClientMethod='get_object',  # 'put_object' works the same way for uploads
    Params={
        'Bucket': BUCKET_NAME,
        'Key': object_key
    },
    ExpiresIn=expiry_seconds
)

print(f"\nPresigned download URL (valid for {expiry_seconds}s):")
print(presigned_download_url)
print(f"\nURL expires at approximately: {datetime.utcnow()} + {expiry_seconds // 60} minutes")

# ── PART 3: Generate a presigned URL for direct-upload from the browser ──
# The client uploads straight to S3 — your server never handles the file bytes.
# This is the standard pattern for large file uploads.
presigned_upload_url = s3_client.generate_presigned_url(
    ClientMethod='put_object',
    Params={
        'Bucket': BUCKET_NAME,
        'Key': 'avatars/user-9001/profile.jpg',
        'ContentType': 'image/jpeg'  # Enforce content type at signing time
    },
    ExpiresIn=300  # Upload must start within 5 minutes
)

print(f"\nPresigned upload URL (valid for 300s):")
print(presigned_upload_url)

import boto3
import json

s3_client = boto3.client('s3', region_name='us-east-1')

BUCKET_NAME = 'acme-corp-user-uploads-dev'

# ── Apply a lifecycle policy that automatically manages storage costs ──
# This policy covers objects under the 'invoices/' prefix only.
# Other prefixes (avatars/, reports/) are not affected.
lifecycle_configuration = {
    "Rules": [
        {
            "ID": "InvoiceArchivalPolicy",
            "Status": "Enabled",  # 'Disabled' to pause without deleting the rule
            "Filter": {
                # Only apply this rule to objects whose key starts with 'invoices/'
                "Prefix": "invoices/"
            },
            "Transitions": [
                {
                    # After 30 days, move to Standard-IA
                    # ~45% cheaper per GB, small per-retrieval fee applies
                    "Days": 30,
                    "StorageClass": "STANDARD_IA"
                },
                {
                    # After 180 days, move to Glacier Instant Retrieval
                    # ~68% cheaper than Standard, millisecond retrieval still available
                    "Days": 180,
                    "StorageClass": "GLACIER_IR"
                },
                {
                    # After 365 days, move to Glacier Deep Archive
                    # Cheapest tier. Retrieval takes up to 12 hours.
                    # Perfect for compliance-required long-term retention
                    "Days": 365,
                    "StorageClass": "DEEP_ARCHIVE"
                }
            ],
            # Permanently delete objects after 7 years (2555 days)
            # Adjust to match your legal/compliance retention requirements
            "Expiration": {
                "Days": 2555
            }
        },
        {
            # Second rule: automatically clean up incomplete multipart uploads
            # Without this, partially uploaded large files silently accumulate costs
            "ID": "CleanupIncompleteMultipartUploads",
            "Status": "Enabled",
            "Filter": {"Prefix": ""},  # Apply to the entire bucket
            "AbortIncompleteMultipartUpload": {
                "DaysAfterInitiation": 7  # Abort uploads not completed within 7 days
            }
        }
    ]
}

response = s3_client.put_bucket_lifecycle_configuration(
    Bucket=BUCKET_NAME,
    LifecycleConfiguration=lifecycle_configuration
)

print(f"Lifecycle policy applied. HTTP status: {response['ResponseMetadata']['HTTPStatusCode']}")

# ── Verify the policy was saved correctly ──
saved_policy = s3_client.get_bucket_lifecycle_configuration(Bucket=BUCKET_NAME)
for rule in saved_policy['Rules']:
    print(f"\nRule ID: {rule['ID']} | Status: {rule['Status']}")
    if 'Transitions' in rule:
        for transition in rule['Transitions']:
            print(f"  → After {transition['Days']} days: {transition['StorageClass']}")
    if 'Expiration' in rule:
        print(f"  → Delete after: {rule['Expiration']['Days']} days")