From d63d2416ff3282cac749669741c8c1651da2166a Mon Sep 17 00:00:00 2001 From: Dmitry Isaenko Date: Wed, 18 Jan 2023 21:12:22 +0300 Subject: [PATCH] Fix BLZ unpack algorithm, cover by test. Few small corrections. --- .../other/System2/ini1/Kip1Unpacker.java | 2 +- .../java/libKonogonka/blz/BlzDecompress.java | 32 +-- .../ctraes/AesCtrBufferedInputStream.java | 12 +- .../package2/Kip1ExtractDecompressedTest.java | 182 ++++++++++++++++++ .../package2/Kip1ExtractTest.java | 6 +- 5 files changed, 211 insertions(+), 23 deletions(-) create mode 100644 src/test/java/libKonogonka/package2/Kip1ExtractDecompressedTest.java diff --git a/src/main/java/libKonogonka/Tools/other/System2/ini1/Kip1Unpacker.java b/src/main/java/libKonogonka/Tools/other/System2/ini1/Kip1Unpacker.java index 0862cc6..f6287c9 100644 --- a/src/main/java/libKonogonka/Tools/other/System2/ini1/Kip1Unpacker.java +++ b/src/main/java/libKonogonka/Tools/other/System2/ini1/Kip1Unpacker.java @@ -112,7 +112,7 @@ public class Kip1Unpacker { if (decompressedLength != sectionDecompressedSize) throw new Exception("Decompression failure. Expected vs. actual decompressed sizes mismatch: " + - decompressedLength + " / " + sectionDecompressedSize); + sectionDecompressedSize + " / " + decompressedLength); return restored; } } diff --git a/src/main/java/libKonogonka/blz/BlzDecompress.java b/src/main/java/libKonogonka/blz/BlzDecompress.java index aaab034..b333e08 100644 --- a/src/main/java/libKonogonka/blz/BlzDecompress.java +++ b/src/main/java/libKonogonka/blz/BlzDecompress.java @@ -38,21 +38,25 @@ public class BlzDecompress { else if (additionalLength < 0) throw new Exception("File not supported. Please file a bug "+additionalLength); - int compressedOffset = compressedAndHeaderSize - headerSize; - int finalOffset = compressedAndHeaderSize + additionalLength; + int delta = compressed.length - compressedAndHeaderSize; + int compressedOffset = compressedAndHeaderSize - headerSize + delta; + int finalOffset = compressedAndHeaderSize + additionalLength + delta; + int resultingSize = finalOffset; /* System.out.printf( - "Additional length : 0x%-8x %d %n" + - "Header size : 0x%-8x %d %n" + - "Compressed+Header size : 0x%-8x %d %n" + - "Compressed offset : 0x%-8x %d %n" + - "Final offset : 0x%-8x %d %n", + "Additional length : %-21d 0x%-8x %n" + + "Additional length : %-21d 0x%-8x %n" + + "Delta : %-21d %n" + + "Compressed+Header size : %-21d 0x%-8x Incorrect: %-8d 0x%-8x %n" + + "Compressed offset : %-21d 0x%-8x Incorrect: %-8d 0x%-8x %n" + + "Resulting Size,Final offset : %-21d 0x%-8x Incorrect: %-8d 0x%-8x %n%n", additionalLength, additionalLength, headerSize, headerSize, - compressedAndHeaderSize, compressedAndHeaderSize, - compressedOffset, compressedOffset, - finalOffset, finalOffset); - */ + delta, + compressed.length, compressed.length, compressedAndHeaderSize, compressedAndHeaderSize, + compressedOffset, compressedOffset, compressedAndHeaderSize - headerSize, compressedAndHeaderSize - headerSize, + finalOffset, finalOffset, compressedAndHeaderSize + additionalLength, compressedAndHeaderSize + additionalLength); + //*/ decompress_loop: while (true){ byte control = compressed[--compressedOffset]; @@ -79,11 +83,13 @@ public class BlzDecompress { decompressed[finalOffset + j] = decompressed[finalOffset + j + segmentPosition]; } control <<= 1; - if (finalOffset == 0) + if (finalOffset == delta) break decompress_loop; } } + if (delta != 0) + System.arraycopy(compressed, 0, decompressed, 0, delta); - return additionalLength+compressedAndHeaderSize; + return resultingSize; } } diff --git a/src/main/java/libKonogonka/ctraes/AesCtrBufferedInputStream.java b/src/main/java/libKonogonka/ctraes/AesCtrBufferedInputStream.java index 732cb5f..dd8d759 100644 --- a/src/main/java/libKonogonka/ctraes/AesCtrBufferedInputStream.java +++ b/src/main/java/libKonogonka/ctraes/AesCtrBufferedInputStream.java @@ -1,5 +1,5 @@ /* - Copyright 2019-2022 Dmitry Isaenko + Copyright 2019-2023 Dmitry Isaenko This file is part of libKonogonka. @@ -32,6 +32,10 @@ public class AesCtrBufferedInputStream extends BufferedInputStream { private final long mediaOffsetPositionEnd; private final long fileSize; + private byte[] decryptedBytes; + private long pseudoPos; + private int pointerInsideDecryptedSection; + public AesCtrBufferedInputStream(AesCtrDecryptForMediaBlocks decryptor, long ncaOffsetPosition, long mediaStartOffset, @@ -49,10 +53,6 @@ public class AesCtrBufferedInputStream extends BufferedInputStream { "\n MediaOffsetPositionEnd "+RainbowDump.formatDecHexString(mediaOffsetPositionEnd)); } - private byte[] decryptedBytes; - private long pseudoPos; - private int pointerInsideDecryptedSection; - @Override public int read(byte[] b, int off, int len) throws IOException { if (off != 0 || len != b.length) @@ -233,7 +233,7 @@ public class AesCtrBufferedInputStream extends BufferedInputStream { @Override public synchronized int read() throws IOException { byte[] b = new byte[1]; - if (read(b) != -1) + if (read(b, 0, 1) != -1) return b[0]; return -1; } diff --git a/src/test/java/libKonogonka/package2/Kip1ExtractDecompressedTest.java b/src/test/java/libKonogonka/package2/Kip1ExtractDecompressedTest.java new file mode 100644 index 0000000..0b2b26e --- /dev/null +++ b/src/test/java/libKonogonka/package2/Kip1ExtractDecompressedTest.java @@ -0,0 +1,182 @@ +package libKonogonka.package2; + +import libKonogonka.Converter; +import libKonogonka.KeyChainHolder; +import libKonogonka.Tools.NCA.NCAProvider; +import libKonogonka.Tools.RomFs.FileSystemEntry; +import libKonogonka.Tools.RomFs.RomFsProvider; +import libKonogonka.Tools.other.System2.System2Provider; +import libKonogonka.Tools.other.System2.ini1.Ini1Provider; +import libKonogonka.Tools.other.System2.ini1.KIP1Provider; +import libKonogonka.ctraes.InFileStreamProducer; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.DisplayName; +import org.junit.jupiter.api.Test; + +import java.io.File; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.stream.Collectors; +import java.util.zip.CRC32; + +/* ..::::::::::::::::::::: # 5 :::::::::::::::::::::.. +* This test validates decompressed KIP1 CRC32 equality and sizes match between reference values and +* 1. Decompressed KIP1 extracted from INI1.bin file +* 2. Decompressed KIP1 extracted from NCA file via streams +* */ + +public class Kip1ExtractDecompressedTest { + final String KEYS_FILE_LOCATION = "FilesForTests"+File.separator+"prod.keys"; + final String XCI_HEADER_KEYS_FILE_LOCATION = "FilesForTests"+File.separator+"xci_header_key.txt"; + + final String pathToFirmware = "FilesForTests"+File.separator+"Firmware 14.1.0"; + + private static KeyChainHolder keyChainHolder; + + final String referenceFat = "FilesForTests"+File.separator+"reference_for_system2"+File.separator+"FAT"+File.separator+"decompressed"; + final String referenceExFat = "FilesForTests"+File.separator+"reference_for_system2"+File.separator+"ExFAT"+File.separator+"decompressed"; + final String exportFat = System.getProperty("java.io.tmpdir")+File.separator+"Exported_FAT"+File.separator+getClass().getSimpleName(); + final String exportExFat = System.getProperty("java.io.tmpdir")+File.separator+"Exported_ExFAT"+File.separator+getClass().getSimpleName(); + + @DisplayName("KIP1 extract test (case 'FS')") + @Test + void testSystem2() throws Exception{ + makeKeys(); + String[] ncaFileNames = collectNcaFileNames(); + List ncaProviders = makeNcaProviders(ncaFileNames); + + NCAProvider system2FatNcaProvider = null; + NCAProvider system2ExFatNcaProvider = null; + + for (NCAProvider ncaProvider : ncaProviders) { + String titleId = Converter.byteArrToHexStringAsLE(ncaProvider.getTitleId()); + if (titleId.equals("0100000000000819")) + system2FatNcaProvider = ncaProvider; + else if (titleId.equals("010000000000081b")) + system2ExFatNcaProvider = ncaProvider; + } + + Assertions.assertNotNull(system2FatNcaProvider); + Assertions.assertNotNull(system2ExFatNcaProvider); + + System.out.println("FAT " + system2FatNcaProvider.getFile().getName()); + System.out.println("ExFAT " + system2ExFatNcaProvider.getFile().getName()); + + Assertions.assertTrue(system2FatNcaProvider.getFile().getName().endsWith("1212c.nca")); + Assertions.assertTrue(system2ExFatNcaProvider.getFile().getName().endsWith("cc081.nca")); + + testExportedFiles(system2FatNcaProvider, exportFat, referenceFat); + testExportedFiles(system2ExFatNcaProvider, exportExFat, referenceExFat); + } + void makeKeys() throws Exception{ + String keyValue = new String(Files.readAllBytes(Paths.get(XCI_HEADER_KEYS_FILE_LOCATION))).trim(); + Assertions.assertNotEquals(0, keyValue.length()); + keyChainHolder = new KeyChainHolder(KEYS_FILE_LOCATION, keyValue); + } + String[] collectNcaFileNames(){ + File firmware = new File(pathToFirmware); + Assertions.assertTrue(firmware.exists()); + String[] ncaFileNames = firmware.list((File directory, String file) -> ( ! file.endsWith(".cnmt.nca") && file.endsWith(".nca"))); + Assertions.assertNotNull(ncaFileNames); + return ncaFileNames; + } + List makeNcaProviders(String[] ncaFileNames) throws Exception{ + List ncaProviders = new ArrayList<>(); + for (String ncaFileName : ncaFileNames){ + File nca = new File(pathToFirmware+File.separator+ncaFileName); + NCAProvider provider = new NCAProvider(nca, keyChainHolder.getRawKeySet()); + ncaProviders.add(provider); + } + + Assertions.assertNotEquals(0, ncaProviders.size()); + + return ncaProviders; + } + + void testExportedFiles(NCAProvider system2NcaProvider, String exportIntoFolder, String referenceFilesFolder) throws Exception{ + RomFsProvider romFsProvider = system2NcaProvider.getNCAContentProvider(0).getRomfs(); + + FileSystemEntry package2FileSystemEntry = romFsProvider.getRootEntry().getContent() + .stream() + .filter(e -> e.getName().equals("nx")) + .collect(Collectors.toList()) + .get(0) + .getContent() + .stream() + .filter(e -> e.getName().equals("package2")) + .collect(Collectors.toList()) + .get(0); + + HashMap referencePathCrc32 = new HashMap<>(); + + Files.list(Paths.get(referenceFilesFolder)) + .filter(file -> file.toString().endsWith(".dec")) + .forEach(path -> referencePathCrc32.put( + path.getFileName().toString().replaceAll("\\..*$", ""), + calculateReferenceCRC32(path))); + System.out.println("Files"); + romFsProvider.exportContent(exportIntoFolder, package2FileSystemEntry); + System2Provider kernelProviderFile = new System2Provider(exportIntoFolder+File.separator+"package2", keyChainHolder); + kernelProviderFile.getIni1Provider().export(exportIntoFolder); + Ini1Provider ini1Provider = new Ini1Provider(Paths.get(exportIntoFolder+File.separator+"INI1.bin")); + for (KIP1Provider kip1Provider : ini1Provider.getKip1List()) { + String kip1Name = kip1Provider.getHeader().getName(); + kip1Provider.exportAsDecompressed(exportIntoFolder); + Path referenceFilePath = Paths.get(referenceFilesFolder+File.separator+kip1Name+".dec"); + Path myFilePath = Paths.get(exportIntoFolder+File.separator+kip1Name+"_decompressed.kip1"); + + System.out.println( + "\nReference : " + referenceFilePath+ + "\nOwn : " + myFilePath); + + validateChecksums(myFilePath, referencePathCrc32.get(kip1Name)); + validateSizes(referenceFilePath, myFilePath); + } + System.out.println("Stream"); + + InFileStreamProducer producer = romFsProvider.getStreamProducer(package2FileSystemEntry); + System2Provider providerStream = new System2Provider(producer, keyChainHolder); + for (KIP1Provider kip1Provider : providerStream.getIni1Provider().getKip1List()){ + String kip1Name = kip1Provider.getHeader().getName(); + kip1Provider.exportAsDecompressed(exportIntoFolder); + Path referenceFilePath = Paths.get(referenceFilesFolder+File.separator+kip1Name+".dec"); + Path myFilePath = Paths.get(exportIntoFolder+File.separator+kip1Name+"_decompressed.kip1"); + + System.out.println( + "\nReference : " + referenceFilePath+ + "\nOwn : " + myFilePath); + + validateChecksums(myFilePath, referencePathCrc32.get(kip1Name)); + validateSizes(referenceFilePath, myFilePath); + } + System.out.println("---"); + } + long calculateReferenceCRC32(Path refPackage2Path){ + try { + byte[] refPackage2Bytes = Files.readAllBytes(refPackage2Path); + CRC32 crc32 = new CRC32(); + crc32.update(refPackage2Bytes, 0, refPackage2Bytes.length); + return crc32.getValue(); + } + catch (Exception e) { + return -1; + } + } + + void validateChecksums(Path myPackage2Path, long refPackage2Crc32) throws Exception{ + // Check CRC32 for package2 file only + byte[] myPackage2Bytes = Files.readAllBytes(myPackage2Path); + CRC32 crc32 = new CRC32(); + crc32.update(myPackage2Bytes, 0, myPackage2Bytes.length); + long myPackage2Crc32 = crc32.getValue(); + Assertions.assertEquals(myPackage2Crc32, refPackage2Crc32); + } + + void validateSizes(Path a, Path b) throws Exception{ + Assertions.assertEquals(Files.size(a), Files.size(b)); + } +} diff --git a/src/test/java/libKonogonka/package2/Kip1ExtractTest.java b/src/test/java/libKonogonka/package2/Kip1ExtractTest.java index d46081e..2f04604 100644 --- a/src/test/java/libKonogonka/package2/Kip1ExtractTest.java +++ b/src/test/java/libKonogonka/package2/Kip1ExtractTest.java @@ -23,9 +23,9 @@ import java.util.stream.Collectors; import java.util.zip.CRC32; /* ..::::::::::::::::::::: # 4 :::::::::::::::::::::.. -* This test validates KIP1.bin CRC32 equality and sizes match between reference values and -* 1. KIP1.bin extracted from INI1.bin file -* 2. KIP1.bin extracted from NCA file via streams +* This test validates KIP1 CRC32 equality and sizes match between reference values and +* 1. KIP1 extracted from INI1.bin file +* 2. KIP1 extracted from NCA file via streams * */ public class Kip1ExtractTest {