From 119b61579727d9872dd814ccf23e99d23e7edd28 Mon Sep 17 00:00:00 2001 From: Dmitry Isaenko Date: Mon, 19 Dec 2022 21:30:58 +0300 Subject: [PATCH] Fix AesCtrBufferedInputStream bug, add few tests --- src/main/java/libKonogonka/Converter.java | 15 +++- .../libKonogonka/Tools/NCA/NCAContent.java | 3 +- .../libKonogonka/Tools/NCA/NCAProvider.java | 6 +- .../libKonogonka/Tools/PFS0/PFS0Header.java | 2 +- .../Tools/RomFs/FileSystemEntry.java | 2 +- .../ctraes/AesCtrBufferedInputStream.java | 14 ++-- .../ctraes/InFileStreamProducer.java | 3 +- .../RomFsDecrypted/ExportNso0FromNcaTest.java | 69 +++++++++++++++++++ .../RomFsDecrypted/NCAProviderSimpleTest.java | 61 ++++++++++++++++ .../RomFsDecrypted/NSODecompressTest.java | 8 +-- .../libKonogonka/RomFsDecrypted/NSOTest.java | 22 +----- .../RomFsDecrypted/NSPpfs0EncryptedTest.java | 7 +- .../RomFsDecrypted/Pfs0EncryptedTest.java | 7 +- .../RomFsDecrypted/RomFsEncryptedTest.java | 7 +- 14 files changed, 187 insertions(+), 39 deletions(-) create mode 100644 src/test/java/libKonogonka/RomFsDecrypted/ExportNso0FromNcaTest.java create mode 100644 src/test/java/libKonogonka/RomFsDecrypted/NCAProviderSimpleTest.java diff --git a/src/main/java/libKonogonka/Converter.java b/src/main/java/libKonogonka/Converter.java index 20935f4..c600ab3 100644 --- a/src/main/java/libKonogonka/Converter.java +++ b/src/main/java/libKonogonka/Converter.java @@ -57,13 +57,26 @@ public class Converter { } public static String intToBinaryString(int value){ - return String.format("%32s", Integer.toBinaryString( value )).replace(' ', '0')+" | "+value; + return String.format("%32s", Integer.toBinaryString( value )).replace(' ', '0'); } public static String longToOctString(long value){ return String.format("%64s", Long.toBinaryString( value )).replace(' ', '0'); } + public static byte[] hexStringToByteArray(String string){ + if (string.length() % 2 != 0) + string = "0" + string; + + int resultSize = string.length() / 2; + byte[] resultingArray = new byte[resultSize]; + + for (int i = 0; i < resultSize; i++){ + resultingArray[i] = (byte) Integer.parseInt(string.substring(i*2, i*2+2), 16); + } + return resultingArray; + } + public static byte[] flip(byte[] bytes){ int size = bytes.length; byte[] ret = new byte[size]; diff --git a/src/main/java/libKonogonka/Tools/NCA/NCAContent.java b/src/main/java/libKonogonka/Tools/NCA/NCAContent.java index fa5e054..0b02496 100644 --- a/src/main/java/libKonogonka/Tools/NCA/NCAContent.java +++ b/src/main/java/libKonogonka/Tools/NCA/NCAContent.java @@ -154,7 +154,8 @@ public class NCAContent { ncaOffsetPosition, mediaStartOffset, mediaEndOffset, - Files.newInputStream(file.toPath())); + Files.newInputStream(file.toPath()), + Files.size(file.toPath())); } else throw new Exception("Crypto type not supported"); diff --git a/src/main/java/libKonogonka/Tools/NCA/NCAProvider.java b/src/main/java/libKonogonka/Tools/NCA/NCAProvider.java index 3eaf6da..0c23929 100644 --- a/src/main/java/libKonogonka/Tools/NCA/NCAProvider.java +++ b/src/main/java/libKonogonka/Tools/NCA/NCAProvider.java @@ -286,7 +286,7 @@ public class NCAProvider { } catch (EmptySectionException ignored){} catch (Exception e){ - log.debug("Unable to get NCA Content "+number, e); + log.debug("Unable to get NCA Content "+number+" ("+file.getParentFile().getName()+"/"+file.getName()+")", e); } } @@ -391,4 +391,8 @@ public class NCAProvider { throw new Exception("NCA Content must be requested in range of 0-3, while 'Section Number "+sectionNumber+"' requested"); } } + + public File getFile() { + return file; + } } \ No newline at end of file diff --git a/src/main/java/libKonogonka/Tools/PFS0/PFS0Header.java b/src/main/java/libKonogonka/Tools/PFS0/PFS0Header.java index 7e20d11..84b95bb 100644 --- a/src/main/java/libKonogonka/Tools/PFS0/PFS0Header.java +++ b/src/main/java/libKonogonka/Tools/PFS0/PFS0Header.java @@ -85,7 +85,7 @@ public class PFS0Header { throw new Exception("Read PFS0Provider String table failure. Can't read requested string table size ("+stringTableSize+")"); } - for (int i=0; i < filesCount; i++){ + for (int i = 0; i < filesCount; i++){ int j = 0; while (stringTbl[strTableOffsets[i]+j] != (byte)0x00) j++; diff --git a/src/main/java/libKonogonka/Tools/RomFs/FileSystemEntry.java b/src/main/java/libKonogonka/Tools/RomFs/FileSystemEntry.java index 75877c2..72cde32 100644 --- a/src/main/java/libKonogonka/Tools/RomFs/FileSystemEntry.java +++ b/src/main/java/libKonogonka/Tools/RomFs/FileSystemEntry.java @@ -196,7 +196,7 @@ public class FileSystemEntry { fileName = new String(Arrays.copyOfRange(filesMetadataTable, i, i + fileNameLength), StandardCharsets.UTF_8); } catch (Exception e){ - log.debug("fileName sizes are: "+filesMetadataTable.length+"\t"+i+"\t"+i + fileNameLength+"\t\t"+nextHashTableBucketFileOffset); + log.debug("fileName sizes are: "+filesMetadataTable.length+"\t"+i+"\t"+i + fileNameLength+"\t\t"+nextHashTableBucketFileOffset, e); } } else { diff --git a/src/main/java/libKonogonka/ctraes/AesCtrBufferedInputStream.java b/src/main/java/libKonogonka/ctraes/AesCtrBufferedInputStream.java index 7a88237..aabe078 100644 --- a/src/main/java/libKonogonka/ctraes/AesCtrBufferedInputStream.java +++ b/src/main/java/libKonogonka/ctraes/AesCtrBufferedInputStream.java @@ -18,6 +18,7 @@ */ package libKonogonka.ctraes; +import libKonogonka.Converter; import libKonogonka.RainbowDump; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; @@ -30,16 +31,19 @@ public class AesCtrBufferedInputStream extends BufferedInputStream { private final AesCtrDecryptSimple decryptor; private final long mediaOffsetPositionStart; private final long mediaOffsetPositionEnd; + private final long fileSize; public AesCtrBufferedInputStream(AesCtrDecryptSimple decryptor, long ncaOffsetPosition, long mediaStartOffset, long mediaEndOffset, - InputStream inputStream){ + InputStream inputStream, + long fileSize){ super(inputStream); this.decryptor = decryptor; this.mediaOffsetPositionStart = ncaOffsetPosition + (mediaStartOffset * 0x200); this.mediaOffsetPositionEnd = ncaOffsetPosition + (mediaEndOffset * 0x200); + this.fileSize = fileSize; log.trace("\n Offset Position "+ncaOffsetPosition+ "\n MediaOffsetPositionStart "+RainbowDump.formatDecHexString(mediaOffsetPositionStart)+ @@ -75,9 +79,11 @@ public class AesCtrBufferedInputStream extends BufferedInputStream { fillDecryptedCache(); System.arraycopy(decryptedBytes, 0, b, bytesFromFirstBlock+i*0x200, 0x200); } - //3 // TODO: ONLY IF NON-NULL?? - fillDecryptedCache(); - System.arraycopy(decryptedBytes, 0, b, bytesFromFirstBlock+middleBlocksCount*0x200, bytesFromLastBlock); + //3 + if(fileSize > (pseudoPos+bytesToRead)) { + fillDecryptedCache(); + System.arraycopy(decryptedBytes, 0, b, bytesFromFirstBlock + middleBlocksCount * 0x200, bytesFromLastBlock); + } pseudoPos += bytesToRead; pointerInsideDecryptedSection = bytesFromLastBlock; return b.length; diff --git a/src/main/java/libKonogonka/ctraes/InFileStreamProducer.java b/src/main/java/libKonogonka/ctraes/InFileStreamProducer.java index b770406..4ab892b 100644 --- a/src/main/java/libKonogonka/ctraes/InFileStreamProducer.java +++ b/src/main/java/libKonogonka/ctraes/InFileStreamProducer.java @@ -70,7 +70,8 @@ public class InFileStreamProducer { initialOffset, mediaStartOffset, mediaEndOffset, - Files.newInputStream(file.toPath())); + Files.newInputStream(file.toPath()), + Files.size(file.toPath())); skipBytesTillBeginning(stream, subOffset); return stream; } diff --git a/src/test/java/libKonogonka/RomFsDecrypted/ExportNso0FromNcaTest.java b/src/test/java/libKonogonka/RomFsDecrypted/ExportNso0FromNcaTest.java new file mode 100644 index 0000000..e8dff26 --- /dev/null +++ b/src/test/java/libKonogonka/RomFsDecrypted/ExportNso0FromNcaTest.java @@ -0,0 +1,69 @@ +/* + Copyright 2018-2022 Dmitry Isaenko + + This file is part of libKonogonka. + + libKonogonka is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + libKonogonka is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with libKonogonka. If not, see . + */ +package libKonogonka.RomFsDecrypted; + +import libKonogonka.KeyChainHolder; +import libKonogonka.Tools.NCA.NCAProvider; +import libKonogonka.Tools.NSO.NSO0Provider; +import libKonogonka.Tools.PFS0.PFS0Provider; +import org.junit.jupiter.api.Disabled; +import org.junit.jupiter.api.DisplayName; +import org.junit.jupiter.api.Test; + +import java.io.BufferedReader; +import java.io.File; +import java.io.FileReader; + +public class ExportNso0FromNcaTest { + private static final String keysFileLocation = "./FilesForTests/prod.keys"; + private static final String xci_header_keyFileLocation = "./FilesForTests/xci_header_key.txt"; + private static final String ncaFileLocation = "./FilesForTests/nso_container.nca"; + private static final String exportDecompressedNsoTo = "/tmp"; + + @Disabled + @DisplayName("Exports decompressed NSO0 example") + @Test + void nso0Test() throws Exception{ + BufferedReader br = new BufferedReader(new FileReader(xci_header_keyFileLocation)); + String keyValue = br.readLine(); + br.close(); + + if (keyValue == null) + throw new Exception("Unable to retrieve xci_header_key"); + + keyValue = keyValue.trim(); + KeyChainHolder keyChainHolder = new KeyChainHolder(keysFileLocation, keyValue); + + NCAProvider ncaProvider = new NCAProvider(new File(ncaFileLocation), keyChainHolder.getRawKeySet()); + + PFS0Provider pfs0Provider = ncaProvider.getNCAContentProvider(0).getPfs0(); + pfs0Provider.printDebug(); + + NSO0Provider nso0Provider = new NSO0Provider(pfs0Provider.getStreamProducer(0)); + nso0Provider.printDebug(); + nso0Provider.exportAsDecompressedNSO0(exportDecompressedNsoTo); + + System.out.println("__--++ SDK VERSION ++--__\n" + + ncaProvider.getSdkVersion()[3] + +"."+ ncaProvider.getSdkVersion()[2] + +"."+ ncaProvider.getSdkVersion()[1] + +"."+ ncaProvider.getSdkVersion()[0]); + + } +} diff --git a/src/test/java/libKonogonka/RomFsDecrypted/NCAProviderSimpleTest.java b/src/test/java/libKonogonka/RomFsDecrypted/NCAProviderSimpleTest.java new file mode 100644 index 0000000..8d92ec7 --- /dev/null +++ b/src/test/java/libKonogonka/RomFsDecrypted/NCAProviderSimpleTest.java @@ -0,0 +1,61 @@ +/* + Copyright 2018-2022 Dmitry Isaenko + + This file is part of libKonogonka. + + libKonogonka is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + libKonogonka is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with libKonogonka. If not, see . + */ +package libKonogonka.RomFsDecrypted; + +import libKonogonka.KeyChainHolder; +import libKonogonka.Tools.NCA.NCAProvider; +import org.junit.jupiter.api.*; + +import java.io.BufferedReader; +import java.io.File; +import java.io.FileReader; + +@TestMethodOrder(MethodOrderer.OrderAnnotation.class) +public class NCAProviderSimpleTest { + private static final String keysFileLocation = "./FilesForTests/prod.keys"; + private static final String xci_header_keyFileLocation = "./FilesForTests/xci_header_key.txt"; + private static final String ncaFileLocation = "./FilesForTests/simple.nca"; + //private static final String ncaFileLocation = "./FilesForTests/4pfs.nca"; + private static KeyChainHolder keyChainHolder; + private static NCAProvider ncaProvider; + + //@Disabled + @Order(1) + @DisplayName("KeyChain loac test") + @Test + void keysChain() throws Exception{ + BufferedReader br = new BufferedReader(new FileReader(xci_header_keyFileLocation)); + String keyValue = br.readLine(); + br.close(); + + if (keyValue == null) + throw new Exception("Unable to retrieve xci_header_key"); + + keyValue = keyValue.trim(); + keyChainHolder = new KeyChainHolder(keysFileLocation, keyValue); + } + + //@Disabled + @Order(2) + @DisplayName("NCA provider test") + @Test + void ncaProvider() throws Exception{ + ncaProvider = new NCAProvider(new File(ncaFileLocation), keyChainHolder.getRawKeySet()); + } +} diff --git a/src/test/java/libKonogonka/RomFsDecrypted/NSODecompressTest.java b/src/test/java/libKonogonka/RomFsDecrypted/NSODecompressTest.java index 7139bba..6704653 100644 --- a/src/test/java/libKonogonka/RomFsDecrypted/NSODecompressTest.java +++ b/src/test/java/libKonogonka/RomFsDecrypted/NSODecompressTest.java @@ -26,18 +26,18 @@ import org.junit.jupiter.api.Test; import java.io.File; public class NSODecompressTest { - private static final String ncaExtractedFileLocation = "./FilesForTests/NSO0/main"; - private static final String ncaExtractedFileLocationDec = "./FilesForTests/NSO0/main_d"; + private static final String nsoExtractedFileLocation = "./FilesForTests/NSO0/main"; + private static final String nsoExtractedFileLocationDec = "./FilesForTests/NSO0/main_d"; @Disabled @DisplayName("NSO0 Decompression test") @Test void nso0DecompressionTest() throws Exception { - NSO0Provider nso0Provider = new NSO0Provider(new File(ncaExtractedFileLocation)); + NSO0Provider nso0Provider = new NSO0Provider(new File(nsoExtractedFileLocation)); //nso0Provider.exportAsDecompressedNSO0("./FilesForTests/NSO0"); nso0Provider.printDebug(); - NSO0Provider nso0Provider1 = new NSO0Provider(new File(ncaExtractedFileLocationDec)); + NSO0Provider nso0Provider1 = new NSO0Provider(new File(nsoExtractedFileLocationDec)); nso0Provider1.printDebug(); } } diff --git a/src/test/java/libKonogonka/RomFsDecrypted/NSOTest.java b/src/test/java/libKonogonka/RomFsDecrypted/NSOTest.java index 1c3d573..b6e2a6f 100644 --- a/src/test/java/libKonogonka/RomFsDecrypted/NSOTest.java +++ b/src/test/java/libKonogonka/RomFsDecrypted/NSOTest.java @@ -24,7 +24,6 @@ import libKonogonka.Tools.NCA.NCAProvider; import libKonogonka.Tools.NSO.NSO0Provider; import libKonogonka.Tools.PFS0.PFS0Provider; import libKonogonka.Tools.PFS0.PFS0subFile; -import libKonogonka.ctraes.AesCtrDecryptSimple; import org.junit.jupiter.api.Disabled; import org.junit.jupiter.api.DisplayName; import org.junit.jupiter.api.Test; @@ -74,19 +73,13 @@ public class NSOTest { } } - long ACBISoffsetPosition; - long ACBISmediaStartOffset; - long ACBISmediaEndOffset; - - long offsetPosition; - - void nso0Validation() throws Exception{ File nca = new File(ncaFileLocation); PFS0subFile[] subfiles = ncaProvider.getNCAContentProvider(0).getPfs0().getHeader().getPfs0subFiles(); - offsetPosition = ncaProvider.getTableEntry0().getMediaStartOffset()*0x200 + + long offsetPosition = ncaProvider.getTableEntry0().getMediaStartOffset()*0x200 + ncaProvider.getNCAContentProvider(0).getPfs0().getRawFileDataStart(); + System.out.println("\t============================================================="); System.out.println("\tNCA SIZE: "+ RainbowDump.formatDecHexString(nca.length())); System.out.println("\tPFS0 Offset(get) "+RainbowDump.formatDecHexString(ncaProvider.getSectionBlock0().getSuperBlockPFS0().getPfs0offset())); @@ -102,15 +95,6 @@ public class NSOTest { } System.out.println("\t============================================================="); - ACBISoffsetPosition = 0; - ACBISmediaStartOffset = ncaProvider.getTableEntry0().getMediaStartOffset(); - ACBISmediaEndOffset = ncaProvider.getTableEntry0().getMediaEndOffset(); - - AesCtrDecryptSimple decryptSimple = new AesCtrDecryptSimple( - ncaProvider.getDecryptedKey2(), - ncaProvider.getSectionBlock0().getSectionCTR(), - ncaProvider.getTableEntry0().getMediaStartOffset() * 0x200); - PFS0Provider pfs0Provider = ncaProvider.getNCAContentProvider(0).getPfs0(); pfs0Provider.printDebug(); @@ -118,7 +102,7 @@ public class NSOTest { nso0Provider.printDebug(); nso0Provider.exportAsDecompressedNSO0("./tmp"); - // NPDMProvider npdmProvider = new NPDMProvider(pfs0Provider.getProviderSubFilePipedInpStream(1)); + //NPDMProvider npdmProvider = new NPDMProvider(pfs0Provider.getStreamProducer(1)); System.out.println("__--++ SDK VERSION ++--__\n" +ncaProvider.getSdkVersion()[3] diff --git a/src/test/java/libKonogonka/RomFsDecrypted/NSPpfs0EncryptedTest.java b/src/test/java/libKonogonka/RomFsDecrypted/NSPpfs0EncryptedTest.java index c171702..1f436b1 100644 --- a/src/test/java/libKonogonka/RomFsDecrypted/NSPpfs0EncryptedTest.java +++ b/src/test/java/libKonogonka/RomFsDecrypted/NSPpfs0EncryptedTest.java @@ -32,6 +32,7 @@ import org.junit.jupiter.api.Test; import java.io.*; import java.nio.file.Files; +import java.nio.file.Path; import java.util.HashMap; public class NSPpfs0EncryptedTest { @@ -153,14 +154,16 @@ public class NSPpfs0EncryptedTest { BufferedOutputStream extractedFileBOS = new BufferedOutputStream(Files.newOutputStream(contentFile.toPath())); //--- - InputStream is = Files.newInputStream(new File(nspFileLocation).toPath()); //TODO: NOTICE + Path filePath = new File(nspFileLocation).toPath(); + InputStream is = Files.newInputStream(filePath); AesCtrBufferedInputStream aesCtrBufferedInputStream = new AesCtrBufferedInputStream( decryptSimple, ACBISoffsetPosition, ACBISmediaStartOffset, ACBISmediaEndOffset, - is); + is, + Files.size(filePath)); //long offsetToSkip = entry.getOffset() + ncaProvider.getNCAContentProvider(0).getPfs0().getRawFileDataStart(); long offsetToSkip = offsetPosition+entry.getOffset(); diff --git a/src/test/java/libKonogonka/RomFsDecrypted/Pfs0EncryptedTest.java b/src/test/java/libKonogonka/RomFsDecrypted/Pfs0EncryptedTest.java index 8d8d150..510bb5f 100644 --- a/src/test/java/libKonogonka/RomFsDecrypted/Pfs0EncryptedTest.java +++ b/src/test/java/libKonogonka/RomFsDecrypted/Pfs0EncryptedTest.java @@ -29,6 +29,7 @@ import org.junit.jupiter.api.*; import java.io.*; import java.nio.file.Files; +import java.nio.file.Path; public class Pfs0EncryptedTest { private static final String keysFileLocation = "./FilesForTests/prod.keys"; @@ -127,14 +128,16 @@ public class Pfs0EncryptedTest { BufferedOutputStream extractedFileBOS = new BufferedOutputStream(Files.newOutputStream(contentFile.toPath())); //--- - InputStream is = Files.newInputStream(new File(ncaFileLocation).toPath()); + Path filePath = new File(ncaFileLocation).toPath(); + InputStream is = Files.newInputStream(filePath); AesCtrBufferedInputStream aesCtrBufferedInputStream = new AesCtrBufferedInputStream( decryptSimple, ACBISoffsetPosition, ACBISmediaStartOffset, ACBISmediaEndOffset, - is); + is, + Files.size(filePath)); //long offsetToSkip = entry.getOffset() + ncaProvider.getNCAContentProvider(0).getPfs0().getRawFileDataStart(); long offsetToSkip = offsetPosition+entry.getOffset(); diff --git a/src/test/java/libKonogonka/RomFsDecrypted/RomFsEncryptedTest.java b/src/test/java/libKonogonka/RomFsDecrypted/RomFsEncryptedTest.java index 5d989bf..fe2b2ca 100644 --- a/src/test/java/libKonogonka/RomFsDecrypted/RomFsEncryptedTest.java +++ b/src/test/java/libKonogonka/RomFsDecrypted/RomFsEncryptedTest.java @@ -29,6 +29,7 @@ import org.junit.jupiter.api.*; import java.io.*; import java.nio.file.Files; +import java.nio.file.Path; @TestMethodOrder(MethodOrderer.OrderAnnotation.class) public class RomFsEncryptedTest { @@ -136,14 +137,16 @@ public class RomFsEncryptedTest { BufferedOutputStream extractedFileBOS = new BufferedOutputStream(Files.newOutputStream(contentFile.toPath())); //--- - InputStream is = Files.newInputStream(new File(ncaFileLocation).toPath()); + Path filePath = new File(ncaFileLocation).toPath(); + InputStream is = Files.newInputStream(filePath); AesCtrBufferedInputStream aesCtrBufferedInputStream = new AesCtrBufferedInputStream( decryptSimple, ACBISoffsetPosition, ACBISmediaStartOffset, ACBISmediaEndOffset, - is); + is, + Files.size(filePath)); long skipBytes = entry.getOffset() +ncaProvider.getTableEntry1().getMediaStartOffset()*0x200