Design Instagram — Hybrid Push-Pull Feed at 100M Followers

package io.thecodeforge.instagram;

import java.util.UUID;
import java.time.Instant;

/**
 * Represents the Metadata record for a high-scale media upload.
 * In production, this would be persisted to a sharded DB cluster.
 */
public class PhotoMetadata {
    private final String photoId;
    private final String userId;
    private final String s3Url;
    private final long timestamp;

    public PhotoMetadata(String userId, String s3Url) {
        this.photoId = UUID.randomUUID().toString();
        this.userId = userId;
        this.s3Url = s3Url;
        this.timestamp = Instant.now().getEpochSecond();
    }

    public void saveToDatabase() {
        // High-level logic for sharded database insertion
        System.out.println("Persisting metadata to shard based on userId: " + userId);
        System.out.println("Photo ID: " + photoId + " | Storage Path: " + s3Url);
    }

    public static void main(String[] args) {
        PhotoMetadata upload = new PhotoMetadata("user_8821", "https://s3.thecodeforge.io/bucket/img_99.jpg");
        upload.saveToDatabase();
    }
}

-- io.thecodeforge.instagram - Database Schema Concepts
-- Sharding Key: user_id

CREATE TABLE io_thecodeforge.users (
    user_id BIGINT PRIMARY KEY,
    username VARCHAR(50) UNIQUE NOT NULL,
    email VARCHAR(100) NOT NULL,
    created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP
);

CREATE TABLE io_thecodeforge.photos (
    photo_id BIGINT PRIMARY KEY,
    user_id BIGINT REFERENCES io_thecodeforge.users(user_id),
    image_path VARCHAR(255) NOT NULL,
    caption TEXT,
    latitude DECIMAL(9,6),
    longitude DECIMAL(9,6),
    created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP
);

-- Index for fast Feed Generation (Sorted by Time)
CREATE INDEX idx_user_photos_time ON io_thecodeforge.photos(user_id, created_at DESC);

package io.thecodeforge.instagram;

import java.util.*;
import java.util.concurrent.*;

public class FeedGenerationService {
    private static final long CELEBRITY_THRESHOLD = 1_000_000;
    private final FollowerService followerService;
    private final FeedCache feedCache;
    private final ExecutorService fanoutExecutor = Executors.newFixedThreadPool(20);

    public void onNewPost(Post post, String userId) {
        long followerCount = followerService.getFollowerCount(userId);
        if (followerCount < CELEBRITY_THRESHOLD) {
            fanoutToAllFollowers(post, userId);
        } else {
            // celebrity: store post for pull-based retrieval
            feedCache.storeCelebrityPost(userId, post);
        }
    }

    private void fanoutToAllFollowers(Post post, String userId) {
        List<String> followers = followerService.getFollowers(userId);
        for (String followerId : followers) {
            fanoutExecutor.submit(() -> feedCache.addToFeed(followerId, post));
        }
    }

    public List<Post> getFeed(String userId, int limit) {
        // merge cached feed with recent posts from followed celebrities
        List<Post> cachedPosts = feedCache.getFeed(userId);
        List<String> followedCelebrities = followerService.getFollowedCelebrities(userId);
        List<Post> celebrityPosts = feedCache.getRecentCelebrityPosts(followedCelebrities);
        List<Post> merged = mergeAndSort(cachedPosts, celebrityPosts);
        return merged.subList(0, Math.min(limit, merged.size()));
    }

    private List<Post> mergeAndSort(List<Post> a, List<Post> b) {
        // merge two sorted (by timestamp descending) lists
        List<Post> result = new ArrayList<>();
        int i = 0, j = 0;
        while (i < a.size() && j < b.size()) {
            result.add(a.get(i).getTimestamp() >= b.get(j).getTimestamp() ? a.get(i++) : b.get(j++));
        }
        while (i < a.size()) result.add(a.get(i++));
        while (j < b.size()) result.add(b.get(j++));
        return result;
    }
}

package io.thecodeforge.instagram;

import java.time.*;

public class StorageStrategy {
    enum StorageTier { HOT, COLD, ARCHIVE }

    public StorageTier classifyMedia(String mediaId, Instant lastAccessTime) {
        long daysSinceLastAccess = Duration.between(lastAccessTime, Instant.now()).toDays();
        if (daysSinceLastAccess <= 30) {
            return StorageTier.HOT;
        } else if (daysSinceLastAccess <= 365) {
            return StorageTier.COLD;
        } else {
            return StorageTier.ARCHIVE;
        }
    }

    // Production: this method would invoke S3 lifecycle policies
    public String getMediaUrl(String mediaId, StorageTier tier) {
        switch(tier) {
            case HOT:
                return "https://s3.us-east-1.amazonaws.com/instagram-hot/" + mediaId;
            case COLD:
                return "https://s3.us-west-2.amazonaws.com/instagram-cold/" + mediaId;
            case ARCHIVE:
                return "https://glacier.amazonaws.com/instagram-archive/" + mediaId;
            default:
                throw new IllegalArgumentException("Unknown tier");
        }
    }

    public static void main(String[] args) {
        StorageStrategy s = new StorageStrategy();
        StorageTier tier = s.classifyMedia("photo_abc123", Instant.now().minus(60, ChronoUnit.DAYS));
        System.out.println("Media URL: " + s.getMediaUrl("photo_abc123", tier));
    }
}

package io.thecodeforge.instagram;

import redis.clients.jedis.Jedis;
import java.util.*;

public class CacheStrategy {
    private static final int FEED_CACHE_SIZE = 1000;
    private final Jedis jedis;

    public CacheStrategy(String redisHost) {
        this.jedis = new Jedis(redisHost);
    }

    public void addPostToUserFeed(String userId, Post post) {
        String key = "feed:" + userId;
        // Use sorted set with timestamp as score
        jedis.zadd(key, post.getTimestamp(), post.getId());
        // Trim to keep only top 1000
        jedis.zremrangeByRank(key, 0, -(FEED_CACHE_SIZE + 1));
    }

    public List<String> getUserFeed(String userId, int start, int count) {
        String key = "feed:" + userId;
        Set<String> ids = jedis.zrevrange(key, start, start + count - 1);
        return new ArrayList<>(ids);
    }

    public long getFeedSize(String userId) {
        return jedis.zcard("feed:" + userId);
    }

    // Cache aside pattern for user profile
    public UserProfile getUserProfile(String userId) {
        String key = "profile:" + userId;
        UserProfile profile = (UserProfile) jedis.get(key); // assuming serialization
        if (profile == null) {
            profile = loadFromDatabase(userId);
            jedis.setex(key, 300, profile.toString()); // 5 min TTL
        }
        return profile;
    }

    private UserProfile loadFromDatabase(String userId) {
        // Placeholder – in reality query sharded DB
        return new UserProfile(userId, "user_" + userId, System.currentTimeMillis());
    }

    public static void main(String[] args) {
        CacheStrategy cache = new CacheStrategy("localhost:6379");
        cache.addPostToUserFeed("user123", new Post("photo1", 1000000L));
        System.out.println("Feed: " + cache.getUserFeed("user123", 0, 10));
    }
}