desu/libKonogonka
1
0
Fork 0
Code Issues Pull requests Projects Releases Wiki Activity

Fix BLZ unpack algorithm, cover by test. Few small corrections.
All checks were successful
continuous-integration/drone/push Build is passing
Details

Browse source
This commit is contained in:
Dmitry Isaenko 2023-01-18 21:12:22 +03:00
parent 0b2548da92
commit d63d2416ff
5 changed files with 211 additions and 23 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
src/main/java/libKonogonka/Tools/other/System2/ini1/Kip1Unpacker.java
View file

@ -112,7 +112,7 @@ public class Kip1Unpacker {
if (decompressedLength != sectionDecompressedSize)
throw new Exception("Decompression failure. Expected vs. actual decompressed sizes mismatch: " +
decompressedLength + " / " + sectionDecompressedSize);
sectionDecompressedSize + " / " + decompressedLength);
return restored;
}
}

32
src/main/java/libKonogonka/blz/BlzDecompress.java
View file

@ -38,21 +38,25 @@ public class BlzDecompress {
else if (additionalLength < 0)
throw new Exception("File not supported. Please file a bug "+additionalLength);
int compressedOffset = compressedAndHeaderSize - headerSize;
int finalOffset = compressedAndHeaderSize + additionalLength;
int delta = compressed.length - compressedAndHeaderSize;
int compressedOffset = compressedAndHeaderSize - headerSize + delta;
int finalOffset = compressedAndHeaderSize + additionalLength + delta;
int resultingSize = finalOffset;
/*
System.out.printf(
"Additional length : 0x%-8x %d %n" +
"Header size : 0x%-8x %d %n" +
"Compressed+Header size : 0x%-8x %d %n" +
"Compressed offset : 0x%-8x %d %n" +
"Final offset : 0x%-8x %d %n",
"Additional length : %-21d 0x%-8x %n" +
"Additional length : %-21d 0x%-8x %n" +
"Delta : %-21d %n" +
"Compressed+Header size : %-21d 0x%-8x Incorrect: %-8d 0x%-8x %n" +
"Compressed offset : %-21d 0x%-8x Incorrect: %-8d 0x%-8x %n" +
"Resulting Size,Final offset : %-21d 0x%-8x Incorrect: %-8d 0x%-8x %n%n",
additionalLength, additionalLength,
headerSize, headerSize,
compressedAndHeaderSize, compressedAndHeaderSize,
compressedOffset, compressedOffset,
finalOffset, finalOffset);
*/
delta,
compressed.length, compressed.length, compressedAndHeaderSize, compressedAndHeaderSize,
compressedOffset, compressedOffset, compressedAndHeaderSize - headerSize, compressedAndHeaderSize - headerSize,
finalOffset, finalOffset, compressedAndHeaderSize + additionalLength, compressedAndHeaderSize + additionalLength);
//*/
decompress_loop:
while (true){
byte control = compressed[--compressedOffset];
@ -79,11 +83,13 @@ public class BlzDecompress {
decompressed[finalOffset + j] = decompressed[finalOffset + j + segmentPosition];
}
control <<= 1;
if (finalOffset == 0)
if (finalOffset == delta)
break decompress_loop;
}
}
if (delta != 0)
System.arraycopy(compressed, 0, decompressed, 0, delta);
return additionalLength+compressedAndHeaderSize;
return resultingSize;
}
}

12
src/main/java/libKonogonka/ctraes/AesCtrBufferedInputStream.java
View file

@ -1,5 +1,5 @@
/*
Copyright 2019-2022 Dmitry Isaenko
Copyright 2019-2023 Dmitry Isaenko
This file is part of libKonogonka.
@ -32,6 +32,10 @@ public class AesCtrBufferedInputStream extends BufferedInputStream {
private final long mediaOffsetPositionEnd;
private final long fileSize;
private byte[] decryptedBytes;
private long pseudoPos;
private int pointerInsideDecryptedSection;
public AesCtrBufferedInputStream(AesCtrDecryptForMediaBlocks decryptor,
long ncaOffsetPosition,
long mediaStartOffset,
@ -49,10 +53,6 @@ public class AesCtrBufferedInputStream extends BufferedInputStream {
"\n MediaOffsetPositionEnd "+RainbowDump.formatDecHexString(mediaOffsetPositionEnd));
}
private byte[] decryptedBytes;
private long pseudoPos;
private int pointerInsideDecryptedSection;
@Override
public int read(byte[] b, int off, int len) throws IOException {
if (off != 0 || len != b.length)
@ -233,7 +233,7 @@ public class AesCtrBufferedInputStream extends BufferedInputStream {
@Override
public synchronized int read() throws IOException {
byte[] b = new byte[1];
if (read(b) != -1)
if (read(b, 0, 1) != -1)
return b[0];
return -1;
}

182
src/test/java/libKonogonka/package2/Kip1ExtractDecompressedTest.java Normal file
View file

@ -0,0 +1,182 @@
package libKonogonka.package2;
import libKonogonka.Converter;
import libKonogonka.KeyChainHolder;
import libKonogonka.Tools.NCA.NCAProvider;
import libKonogonka.Tools.RomFs.FileSystemEntry;
import libKonogonka.Tools.RomFs.RomFsProvider;
import libKonogonka.Tools.other.System2.System2Provider;
import libKonogonka.Tools.other.System2.ini1.Ini1Provider;
import libKonogonka.Tools.other.System2.ini1.KIP1Provider;
import libKonogonka.ctraes.InFileStreamProducer;
import org.junit.jupiter.api.Assertions;
import org.junit.jupiter.api.DisplayName;
import org.junit.jupiter.api.Test;
import java.io.File;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.stream.Collectors;
import java.util.zip.CRC32;
/* ..::::::::::::::::::::: # 5 :::::::::::::::::::::..
* This test validates decompressed KIP1 CRC32 equality and sizes match between reference values and
* 1. Decompressed KIP1 extracted from INI1.bin file
* 2. Decompressed KIP1 extracted from NCA file via streams
* */
public class Kip1ExtractDecompressedTest {
final String KEYS_FILE_LOCATION = "FilesForTests"+File.separator+"prod.keys";
final String XCI_HEADER_KEYS_FILE_LOCATION = "FilesForTests"+File.separator+"xci_header_key.txt";
final String pathToFirmware = "FilesForTests"+File.separator+"Firmware 14.1.0";
private static KeyChainHolder keyChainHolder;
final String referenceFat = "FilesForTests"+File.separator+"reference_for_system2"+File.separator+"FAT"+File.separator+"decompressed";
final String referenceExFat = "FilesForTests"+File.separator+"reference_for_system2"+File.separator+"ExFAT"+File.separator+"decompressed";
final String exportFat = System.getProperty("java.io.tmpdir")+File.separator+"Exported_FAT"+File.separator+getClass().getSimpleName();
final String exportExFat = System.getProperty("java.io.tmpdir")+File.separator+"Exported_ExFAT"+File.separator+getClass().getSimpleName();
@DisplayName("KIP1 extract test (case 'FS')")
@Test
void testSystem2() throws Exception{
makeKeys();
String[] ncaFileNames = collectNcaFileNames();
List<NCAProvider> ncaProviders = makeNcaProviders(ncaFileNames);
NCAProvider system2FatNcaProvider = null;
NCAProvider system2ExFatNcaProvider = null;
for (NCAProvider ncaProvider : ncaProviders) {
String titleId = Converter.byteArrToHexStringAsLE(ncaProvider.getTitleId());
if (titleId.equals("0100000000000819"))
system2FatNcaProvider = ncaProvider;
else if (titleId.equals("010000000000081b"))
system2ExFatNcaProvider = ncaProvider;
}
Assertions.assertNotNull(system2FatNcaProvider);
Assertions.assertNotNull(system2ExFatNcaProvider);
System.out.println("FAT " + system2FatNcaProvider.getFile().getName());
System.out.println("ExFAT " + system2ExFatNcaProvider.getFile().getName());
Assertions.assertTrue(system2FatNcaProvider.getFile().getName().endsWith("1212c.nca"));
Assertions.assertTrue(system2ExFatNcaProvider.getFile().getName().endsWith("cc081.nca"));
testExportedFiles(system2FatNcaProvider, exportFat, referenceFat);
testExportedFiles(system2ExFatNcaProvider, exportExFat, referenceExFat);
}
void makeKeys() throws Exception{
String keyValue = new String(Files.readAllBytes(Paths.get(XCI_HEADER_KEYS_FILE_LOCATION))).trim();
Assertions.assertNotEquals(0, keyValue.length());
keyChainHolder = new KeyChainHolder(KEYS_FILE_LOCATION, keyValue);
}
String[] collectNcaFileNames(){
File firmware = new File(pathToFirmware);
Assertions.assertTrue(firmware.exists());
String[] ncaFileNames = firmware.list((File directory, String file) -> ( ! file.endsWith(".cnmt.nca") && file.endsWith(".nca")));
Assertions.assertNotNull(ncaFileNames);
return ncaFileNames;
}
List<NCAProvider> makeNcaProviders(String[] ncaFileNames) throws Exception{
List<NCAProvider> ncaProviders = new ArrayList<>();
for (String ncaFileName : ncaFileNames){
File nca = new File(pathToFirmware+File.separator+ncaFileName);
NCAProvider provider = new NCAProvider(nca, keyChainHolder.getRawKeySet());
ncaProviders.add(provider);
}
Assertions.assertNotEquals(0, ncaProviders.size());
return ncaProviders;
}
void testExportedFiles(NCAProvider system2NcaProvider, String exportIntoFolder, String referenceFilesFolder) throws Exception{
RomFsProvider romFsProvider = system2NcaProvider.getNCAContentProvider(0).getRomfs();
FileSystemEntry package2FileSystemEntry = romFsProvider.getRootEntry().getContent()
.stream()
.filter(e -> e.getName().equals("nx"))
.collect(Collectors.toList())
.get(0)
.getContent()
.stream()
.filter(e -> e.getName().equals("package2"))
.collect(Collectors.toList())
.get(0);
HashMap<String, Long> referencePathCrc32 = new HashMap<>();
Files.list(Paths.get(referenceFilesFolder))
.filter(file -> file.toString().endsWith(".dec"))
.forEach(path -> referencePathCrc32.put(
path.getFileName().toString().replaceAll("\\..*$", ""),
calculateReferenceCRC32(path)));
System.out.println("Files");
romFsProvider.exportContent(exportIntoFolder, package2FileSystemEntry);
System2Provider kernelProviderFile = new System2Provider(exportIntoFolder+File.separator+"package2", keyChainHolder);
kernelProviderFile.getIni1Provider().export(exportIntoFolder);
Ini1Provider ini1Provider = new Ini1Provider(Paths.get(exportIntoFolder+File.separator+"INI1.bin"));
for (KIP1Provider kip1Provider : ini1Provider.getKip1List()) {
String kip1Name = kip1Provider.getHeader().getName();
kip1Provider.exportAsDecompressed(exportIntoFolder);
Path referenceFilePath = Paths.get(referenceFilesFolder+File.separator+kip1Name+".dec");
Path myFilePath = Paths.get(exportIntoFolder+File.separator+kip1Name+"_decompressed.kip1");
System.out.println(
"\nReference : " + referenceFilePath+
"\nOwn : " + myFilePath);
validateChecksums(myFilePath, referencePathCrc32.get(kip1Name));
validateSizes(referenceFilePath, myFilePath);
}
System.out.println("Stream");
InFileStreamProducer producer = romFsProvider.getStreamProducer(package2FileSystemEntry);
System2Provider providerStream = new System2Provider(producer, keyChainHolder);
for (KIP1Provider kip1Provider : providerStream.getIni1Provider().getKip1List()){
String kip1Name = kip1Provider.getHeader().getName();
kip1Provider.exportAsDecompressed(exportIntoFolder);
Path referenceFilePath = Paths.get(referenceFilesFolder+File.separator+kip1Name+".dec");
Path myFilePath = Paths.get(exportIntoFolder+File.separator+kip1Name+"_decompressed.kip1");
System.out.println(
"\nReference : " + referenceFilePath+
"\nOwn : " + myFilePath);
validateChecksums(myFilePath, referencePathCrc32.get(kip1Name));
validateSizes(referenceFilePath, myFilePath);
}
System.out.println("---");
}
long calculateReferenceCRC32(Path refPackage2Path){
try {
byte[] refPackage2Bytes = Files.readAllBytes(refPackage2Path);
CRC32 crc32 = new CRC32();
crc32.update(refPackage2Bytes, 0, refPackage2Bytes.length);
return crc32.getValue();
}
catch (Exception e) {
return -1;
}
}
void validateChecksums(Path myPackage2Path, long refPackage2Crc32) throws Exception{
// Check CRC32 for package2 file only
byte[] myPackage2Bytes = Files.readAllBytes(myPackage2Path);
CRC32 crc32 = new CRC32();
crc32.update(myPackage2Bytes, 0, myPackage2Bytes.length);
long myPackage2Crc32 = crc32.getValue();
Assertions.assertEquals(myPackage2Crc32, refPackage2Crc32);
}
void validateSizes(Path a, Path b) throws Exception{
Assertions.assertEquals(Files.size(a), Files.size(b));
}
}

6
src/test/java/libKonogonka/package2/Kip1ExtractTest.java
View file

@ -23,9 +23,9 @@ import java.util.stream.Collectors;
import java.util.zip.CRC32;
/* ..::::::::::::::::::::: # 4 :::::::::::::::::::::..
* This test validates KIP1.bin CRC32 equality and sizes match between reference values and
* 1. KIP1.bin extracted from INI1.bin file
* 2. KIP1.bin extracted from NCA file via streams
* This test validates KIP1 CRC32 equality and sizes match between reference values and
* 1. KIP1 extracted from INI1.bin file
* 2. KIP1 extracted from NCA file via streams
* */
public class Kip1ExtractTest {