"A local file header is corrupt" error after upgrading to 3.0/3.1 #1094
Description
We have an application that reads and process information from zip files. After we upgraded target framework to 3.0 (or 3.1) from 2.1 without upgrading the code, processing of larger zip files started to fail with:
System.IO.InvalidDataException: A local file header is corrupt.
This issue blocks us from upgrading to dotnet 3.1 and we need to upgrade asap to resolve the out of memory issues 2.1 has while running in container. I spent some time narrowing down the issue and here's what I found...
Sample code to repro the issue is:
using System;
using System.IO;
using System.IO.Compression;
namespace ZipProblemRepro
{
class Program
{
private const string ZipPath = "F:/Test/bad_header/java_1_file.zip";
public static void Main(string[] args)
{
Console.WriteLine($"Starting to read '{ZipPath}'...");
using (var zip = ZipFile.Open(ZipPath, ZipArchiveMode.Read))
{
var linesProcessed = 0L;
foreach (var fileEntry in zip.Entries)
{
using (var stream = fileEntry.Open())
{
linesProcessed += ReadLines(stream);
}
}
Console.WriteLine($"Processed {linesProcessed} lines");
}
Console.WriteLine("Completed successfully!");
}
private static int ReadLines(Stream stream)
{
var linesRead = 0;
using (var reader = new StreamReader(stream))
{
string line;
while (!reader.EndOfStream)
{
try
{
line = reader.ReadLine();
} catch (OutOfMemoryException ex)
{
line = null;
}
++linesRead;
}
}
return linesRead;
}
}
}While running this code on the same zip file, it succeeds when target framework is netcoreapp2.1, but fails when targeting netcoreapp3.0 or netcoreapp3.1. All tests were performed on the same machine with .NET Core SDK 3.1.100
The hardest part was to produce a brand new zip to repro the problem. Turned out that 2 things are required to get a repro zip:
- Zip should contain a large file in it
- Zip should be created with Java zip library. This sounds weird, but this is the only way I was able to repro this. Tried different settings for 7z, Windows Explorer compression and command line
zip, but no luck with any of those.
I am including the code I used to generate repro zip below, but I can also supply this zip if you tell me where to upload it (315MB).
C# code I used to generate repro file:
using System;
using System.IO;
using System.IO.Compression;
using System.Text;
namespace ZipGenerator
{
class Program
{
private const string TempDirPath = "Temp";
private const int GuildsPerLine = 10;
private const int LinesPerFile = 12000000;
private const int FilesPerDir = 1;
private const int DirsTotal = 1;
public static void Main(string[] args)
{
Console.WriteLine("Generating...");
Directory.CreateDirectory(TempDirPath);
for (var dirCount = 0; dirCount < DirsTotal; dirCount++)
{
if (dirCount % 1 == 0)
{
Console.WriteLine($"Generated {dirCount} out of {DirsTotal} directories so far...");
}
var dirName = Guid.NewGuid().ToString("N");
var dirPath = Path.Combine(TempDirPath, dirName);
Directory.CreateDirectory(dirPath);
for (var fileCount = 0; fileCount < FilesPerDir; fileCount++)
{
var fileName = Guid.NewGuid().ToString("N");
var filePath = Path.Combine(dirPath, fileName);
GenerateEasierToCompressFileWithGuids(filePath);
}
}
Console.WriteLine("Files created!");
}
private static void GenerateEasierToCompressFileWithGuids(string filePath)
{
using (var file = new StreamWriter(filePath))
{
for (var lineCount = 0; lineCount < LinesPerFile; lineCount++)
{
var sb = new StringBuilder();
var fixedGuidPerLine = Guid.NewGuid().ToString("N");
for (var guidInLineCount = 0; guidInLineCount < GuildsPerLine; guidInLineCount++)
{
sb.Append(fixedGuidPerLine);
}
file.WriteLine(sb.ToString());
}
}
}
}
}Java code to zip the file:
package com.test.zipper;
import java.io.*;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.util.zip.Deflater;
import java.util.zip.ZipEntry;
import java.util.zip.ZipOutputStream;
public class Main {
private static final String dirToZip = "D:\\VS\\Sandbox\\ZipProblemRepro\\Temp";
public static void main(String[] args) throws IOException {
String outputFile = "F:\\Test\\bad_header\\javaOutput.zip";
System.out.println("Zipping everything in '" + dirToZip + "'");
FileOutputStream fos = new FileOutputStream(outputFile);
BufferedOutputStream bos = new BufferedOutputStream(fos);
ZipOutputStream zipOutputStream = new ZipOutputStream(bos);
zipOutputStream.setLevel(Deflater.BEST_SPEED);
Files.walk(Paths.get(dirToZip))
.filter(Files::isRegularFile)
.forEach((path) -> ZipFile(zipOutputStream, path));
System.out.println("Done zipping. Closing everything...");
zipOutputStream.close();
bos.close();
fos.close();
System.out.println("Done!");
}
private static void ZipFile(ZipOutputStream zipOutputStream, Path path) {
try {
String fileToZip = path.toAbsolutePath().toString();
String pathInZip = fileToZip.substring(dirToZip.length() + 1);
FileInputStream fis = new FileInputStream(fileToZip);
ZipEntry zipEntry = new ZipEntry(pathInZip);
zipOutputStream.putNextEntry(zipEntry);
byte[] bytes = new byte[1024];
int length;
while((length = fis.read(bytes)) >= 0) {
zipOutputStream.write(bytes, 0, length);
}
zipOutputStream.closeEntry();
} catch (Exception ex) {
throw new RuntimeException("Failed while zipping. Exception was: " + ex.getMessage());
}
}
}