feat(validation): Enhance WARC validation for gzip files with multi-member support

Author: lfoppiano

src/main/java/org/commoncrawl/whirlwind/ValidateWARC.java

@@ -23,9 +23,14 @@
 import java.io.InputStream;
 import java.nio.file.Files;
 import java.nio.file.Path;
-import java.util.concurrent.atomic.AtomicInteger;
 
 public class ValidateWARC {
+
+	private static final int READ_BUFFER_SIZE = 64 * 1024;
+
+	private static class StopReadingException extends IOException {
+	}
+
 	public static void main(String[] args) throws Exception {
 		if (args.length != 1) {
 			System.err.println("Usage: java ValidateWARC <file.gz>");
@@ -38,41 +43,62 @@ public static void main(String[] args) throws Exception {
 		}
 
 		int n = getWarcCompressionInformation(requested);
-		if (n <= 1) {
-			System.out.println("Single-member gzip (likely whole-file gzip). members=" + n);
+		if (n == 0) {
+			System.out.println("No gzip members found (empty or not a gzip file). members=0");
+		} else if (n == 1) {
+			System.out.println("Single-member gzip (likely whole-file gzip). members=1");
 		} else {
 			System.out.println("Concatenated multi-member gzip (record-compressed). members=" + n);
 		}
 
 	}
 
-	public static int getWarcCompressionInformation(Path inputWarc) throws IOException {
-		final AtomicInteger memberCount = new AtomicInteger(0);
+	/**
+	 * Counts gzip members in the file. If stopAfter > 0, stops early once that many
+	 * members have been found (avoids decompressing the entire file). Pass
+	 * stopAfter = 0 to count all members.
+	 */
+	private static int countGzipMembers(Path inputWarc, int stopAfter) throws IOException {
+		int[] memberCount = { 0 };
 
 		try (InputStream fis = Files.newInputStream(inputWarc);
 				BufferedInputStream bis = new BufferedInputStream(fis);
 				GzipCompressorInputStream gz = GzipCompressorInputStream.builder().setDecompressConcatenated(true)
-						.setOnMemberEnd(x -> memberCount.incrementAndGet()).setInputStream(bis).get()) {
+						.setOnMemberEnd(x -> {
+							memberCount[0]++;
+							if (stopAfter > 0 && memberCount[0] >= stopAfter) {
+								throw new StopReadingException();
+							}
+						}).setInputStream(bis).get()) {
 
-			byte[] buf = new byte[64 * 1024];
+			byte[] buf = new byte[READ_BUFFER_SIZE];
 			while (gz.read(buf) != -1) {
-				// Read the entire stream to trigger member processing
-				// We might not need to read the whole stream, just enough to get an idea
+				// drain the stream to trigger member callbacks
 			}
-		} catch (IOException e) {
-			throw new IllegalArgumentException("The file is either not a gzip file or is corrupted.", e);
+		} catch (StopReadingException e) {
+			// early exit — threshold reached
 		}
 
-		return memberCount.get();
+		return memberCount[0];
 	}
 
-	public static void validateRandomAccessWarcOrFail(Path inputWarc) throws IOException {
-		int n = getWarcCompressionInformation(inputWarc);
+	public static int getWarcCompressionInformation(Path inputWarc) throws IOException {
+		return countGzipMembers(inputWarc, 0);
+	}
 
-		if (n <= 1) {
-			throw new IOException(
-					"Non-chunked gzip file detected, gzip block continues\n" + "    beyond single record. " + n);
-		}
+	/**
+	 * Fast check: returns true if the gzip file contains more than one member.
+	 * Stops reading as soon as the second member is detected, avoiding full
+	 * decompression of large files.
+	 */
+	public static boolean isMultiMemberGzip(Path inputWarc) throws IOException {
+		return countGzipMembers(inputWarc, 2) >= 2;
+	}
 
+	public static void validateRandomAccessWarcOrFail(Path inputWarc) throws IOException {
+		if (!isMultiMemberGzip(inputWarc)) {
+			throw new IOException("Non-chunked gzip file detected, gzip block continues\n"
+					+ "    beyond single record. File must be record-compressed (multi-member gzip).");
+		}
 	}
 }