Java StringTokenizer Explained — How It Works, When to Use It, and When to Avoid It

import java.util.StringTokenizer;

public class BasicTokenizerDemo {

    public static void main(String[] args) {

        // A raw HTTP query string — the kind you'd parse from a URL
        String queryString = "user=alice&role=admin&theme=dark&lang=en";

        // Create a tokenizer that splits on '&' characters
        // The second argument is the delimiter set — every char in it is a delimiter
        StringTokenizer tokenizer = new StringTokenizer(queryString, "&");

        System.out.println("Parsing query string: " + queryString);
        System.out.println("Number of tokens found: " + tokenizer.countTokens());
        System.out.println();

        // hasMoreTokens() returns false the moment the cursor hits the end
        while (tokenizer.hasMoreTokens()) {
            String token = tokenizer.nextToken(); // advances the internal cursor
            System.out.println("  Token: " + token);
        }

        System.out.println();
        System.out.println("Any tokens left? " + tokenizer.hasMoreTokens()); // false
    }
}

import java.util.StringTokenizer;
import java.util.LinkedHashMap;
import java.util.Map;

public class QueryStringParser {

    /**
     * Parses a URL query string like "name=alice&age=30&city=london"
     * into a proper key-value Map.
     */
    public static Map<String, String> parse(String queryString) {
        Map<String, String> params = new LinkedHashMap<>();

        // Using '&' and '=' as delimiters — every char here is treated separately
        StringTokenizer tokenizer = new StringTokenizer(queryString, "&=");

        // Tokens now come out in order: key, value, key, value...
        while (tokenizer.hasMoreTokens()) {
            String key = tokenizer.nextToken();   // e.g. "name"
            if (!tokenizer.hasMoreTokens()) break; // guard against malformed input
            String value = tokenizer.nextToken(); // e.g. "alice"
            params.put(key, value);
        }

        return params;
    }

    public static void main(String[] args) {
        String rawQuery = "name=alice&age=30&city=london&premium=true";

        Map<String, String> result = parse(rawQuery);

        System.out.println("Parsed query parameters:");
        result.forEach((key, value) ->
            System.out.printf("  %-10s => %s%n", key, value)
        );
    }
}

import java.util.StringTokenizer;
import java.util.Arrays;

public class TokenizerVsSplit {

    public static void main(String[] args) {

        // A CSV line with an empty field (two consecutive commas)
        String csvLine = "alice,30,,london,true";

        System.out.println("=== String.split() behavior ===");
        // split() respects the empty token between the two commas
        String[] splitResult = csvLine.split(",");
        System.out.println("Token count: " + splitResult.length);
        for (int i = 0; i < splitResult.length; i++) {
            System.out.printf("  [%d] = '%s'%n", i, splitResult[i]);
        }

        System.out.println();
        System.out.println("=== StringTokenizer behavior ===");
        // StringTokenizer silently skips the empty field between double commas
        StringTokenizer tokenizer = new StringTokenizer(csvLine, ",");
        System.out.println("Token count: " + tokenizer.countTokens());
        int index = 0;
        while (tokenizer.hasMoreTokens()) {
            System.out.printf("  [%d] = '%s'%n", index++, tokenizer.nextToken());
        }

        System.out.println();
        System.out.println("Key insight: StringTokenizer lost the empty field.");
        System.out.println("For real CSV parsing, split() or a library is safer.");
    }
}

import java.util.StringTokenizer;
import java.util.ArrayList;
import java.util.List;

public class LogParser {

    // Represents a single parsed log entry
    record LogEntry(String timestamp, String level, String thread, String message) {}

    /**
     * Parses log lines in the format:
     * 2024-01-15T10:23:01|ERROR|http-worker-3|Connection pool exhausted
     */
    public static List<LogEntry> parseErrors(List<String> rawLines) {
        List<LogEntry> errorEntries = new ArrayList<>();

        for (String line : rawLines) {
            // Pipe is the delimiter — simple character, perfect for StringTokenizer
            StringTokenizer tokenizer = new StringTokenizer(line, "|");

            // Guard: a valid log line must have exactly 4 fields
            if (tokenizer.countTokens() != 4) {
                System.out.println("Skipping malformed line: " + line);
                continue;
            }

            String timestamp = tokenizer.nextToken();
            String level     = tokenizer.nextToken();
            String thread    = tokenizer.nextToken();
            String message   = tokenizer.nextToken();

            // Only collect ERROR-level entries
            if ("ERROR".equals(level)) {
                errorEntries.add(new LogEntry(timestamp, level, thread, message));
            }
        }

        return errorEntries;
    }

    public static void main(String[] args) {
        List<String> sampleLog = List.of(
            "2024-01-15T10:23:00|INFO|main|Application started",
            "2024-01-15T10:23:01|ERROR|http-worker-3|Connection pool exhausted",
            "2024-01-15T10:23:02|WARN|scheduler-1|Job queue is 80% full",
            "2024-01-15T10:23:03|ERROR|http-worker-1|Timeout waiting for DB response",
            "CORRUPTED LINE WITHOUT PROPER FORMAT",
            "2024-01-15T10:23:05|INFO|main|Graceful shutdown initiated"
        );

        List<LogEntry> errors = parseErrors(sampleLog);

        System.out.println("\n--- ERROR Log Entries ---");
        for (LogEntry entry : errors) {
            System.out.printf("[%s] (%s) %s%n",
                entry.timestamp(), entry.thread(), entry.message());
        }
        System.out.println("Total errors found: " + errors.size());
    }
}