From f058905e2fb808e6b010dcb8d32a942ceb587bbc Mon Sep 17 00:00:00 2001 From: Dmitry Isaenko Date: Mon, 19 Sep 2022 23:04:59 +0300 Subject: [PATCH] Misc fixes, start rewriting RomFS parts to make it in line with AesCtrBufferedInputStream --- README.md | 6 +- src/main/java/libKonogonka/RainbowDump.java | 52 ++++- .../libKonogonka/Tools/NCA/NCAContent.java | 14 +- .../libKonogonka/Tools/NCA/NCAProvider.java | 2 - .../libKonogonka/Tools/PFS0/PFS0Provider.java | 66 ++++++- .../Tools/RomFs/Level6Header.java | 8 +- .../Tools/RomFs/RomFsConstruct.java | 160 ++++++++++++++++ .../Tools/RomFs/RomFsContentRetrieve.java | 125 ++++++++++++ .../Tools/RomFs/RomFsDecryptedConstruct.java | 121 ------------ .../RomFs/RomFsDecryptedContentRetrieve.java | 93 --------- .../Tools/RomFs/RomFsDecryptedProvider.java | 76 -------- .../Tools/RomFs/RomFsEncryptedConstruct.java | 180 ------------------ .../RomFs/RomFsEncryptedContentRetrieve.java | 139 -------------- ...ryptedProvider.java => RomFsProvider.java} | 97 ++++++---- .../libKonogonka/Tools/XCI/HFS0Provider.java | 131 ++++++------- .../libKonogonka/Tools/XCI/XCIProvider.java | 4 +- .../ctraes/AesCtrBufferedInputStream.java | 27 ++- .../RomFsDecrypted/RomFsDecryptedTest.java | 6 +- .../RomFsDecrypted/RomFsEncryptedTest.java | 4 +- .../libKonogonka/RomFsDecrypted/XciTest.java | 66 +++++++ 20 files changed, 619 insertions(+), 758 deletions(-) create mode 100644 src/main/java/libKonogonka/Tools/RomFs/RomFsConstruct.java create mode 100644 src/main/java/libKonogonka/Tools/RomFs/RomFsContentRetrieve.java delete mode 100644 src/main/java/libKonogonka/Tools/RomFs/RomFsDecryptedConstruct.java delete mode 100644 src/main/java/libKonogonka/Tools/RomFs/RomFsDecryptedContentRetrieve.java delete mode 100644 src/main/java/libKonogonka/Tools/RomFs/RomFsDecryptedProvider.java delete mode 100644 src/main/java/libKonogonka/Tools/RomFs/RomFsEncryptedConstruct.java delete mode 100644 src/main/java/libKonogonka/Tools/RomFs/RomFsEncryptedContentRetrieve.java rename src/main/java/libKonogonka/Tools/RomFs/{RomFsEncryptedProvider.java => RomFsProvider.java} (54%) create mode 100644 src/test/java/libKonogonka/RomFsDecrypted/XciTest.java diff --git a/README.md b/README.md index 4138033..8d09e32 100644 --- a/README.md +++ b/README.md @@ -1,12 +1,12 @@ # libKonogonka -[![Build Status](https://ci.redrise.ru/api/badges/desu/libKonogonka/status.svg)](https://ci.redrise.ru/desu/libKonogonka) +![License](https://img.shields.io/badge/License-GPLv3-blue.svg) [![Build Status](https://ci.redrise.ru/api/badges/desu/libKonogonka/status.svg)](https://ci.redrise.ru/desu/libKonogonka) -Library to work with NS-specific files / filesystem images. Dedicated back end part of [konogonka](https://github.com/developersu/konogonka) ([independent src location](https://git.redrise.ru/desu/konogonka)) +Library to work with NS-specific files / filesystem images. Ex-backend of [konogonka](https://github.com/developersu/konogonka) ([independent src location](https://git.redrise.ru/desu/konogonka)) ### License -[GNU General Public License v3+](https://git.redrise.ru/desu/libKonogonka/LICENSE) +[GNU General Public License v3 or higher](https://git.redrise.ru/desu/libKonogonka/LICENSE) ### Used libraries & resources * [Bouncy Castle](https://www.bouncycastle.org/) for Java. diff --git a/src/main/java/libKonogonka/RainbowDump.java b/src/main/java/libKonogonka/RainbowDump.java index 02a05e8..997e680 100644 --- a/src/main/java/libKonogonka/RainbowDump.java +++ b/src/main/java/libKonogonka/RainbowDump.java @@ -36,14 +36,62 @@ public class RainbowDump { public static void hexDumpUTF8(byte[] byteArray){ + if (byteArray == null || byteArray.length == 0) + return; + + int k = 0; + System.out.printf("%s%08x %s", ANSI_BLUE, 0, ANSI_RESET); + for (int i = 0; i < byteArray.length; i++) { + if (k == 8) + System.out.print(" "); + if (k == 16){ + System.out.print(ANSI_GREEN+"| "+ANSI_RESET); + printChars(byteArray, i); + System.out.println(); + System.out.printf("%s%08x %s", ANSI_BLUE, i, ANSI_RESET); + k = 0; + } + System.out.printf("%02x ", byteArray[i]); + k++; + } + int paddingSize = 16 - (byteArray.length % 16); + if (paddingSize != 16) { + for (int i = 0; i < paddingSize; i++) { + System.out.print(" "); + } + if (paddingSize > 7) { + System.out.print(" "); + } + } + System.out.print(ANSI_GREEN+"| "+ANSI_RESET); + printChars(byteArray, byteArray.length); + System.out.println(); + System.out.print(ANSI_RESET+new String(byteArray, StandardCharsets.UTF_8)+"\n"); + } + + private static void printChars(byte[] byteArray, int pointer){ + for (int j = pointer-16; j < pointer; j++){ + if ((byteArray[j] > 21) && (byteArray[j] < 126)) // man ascii + System.out.print((char) byteArray[j]); + else if (byteArray[j] == 0x0a) + System.out.print("↲"); //"␤" + else if (byteArray[j] == 0x0d) + System.out.print("←"); // "␍" + else + System.out.print("."); + } + } + + + public static void hexDumpUTF8Legacy(byte[] byteArray){ if (byteArray == null || byteArray.length == 0) return; System.out.print(ANSI_BLUE); for (int i=0; i < byteArray.length; i++) - System.out.print(String.format("%02d-", i%100)); + System.out.printf("%02d-", i%100); System.out.println(">"+ANSI_RED+byteArray.length+ANSI_RESET); for (byte b: byteArray) - System.out.print(String.format("%02x ", b)); + System.out.printf("%02x ", b); System.out.println(); System.out.print(new String(byteArray, StandardCharsets.UTF_8)+"\n"); } diff --git a/src/main/java/libKonogonka/Tools/NCA/NCAContent.java b/src/main/java/libKonogonka/Tools/NCA/NCAContent.java index 56a1910..8de079d 100644 --- a/src/main/java/libKonogonka/Tools/NCA/NCAContent.java +++ b/src/main/java/libKonogonka/Tools/NCA/NCAContent.java @@ -22,7 +22,7 @@ import libKonogonka.Tools.NCA.NCASectionTableBlock.NcaFsHeader; import libKonogonka.Tools.PFS0.IPFS0Provider; import libKonogonka.Tools.PFS0.PFS0Provider; import libKonogonka.Tools.RomFs.IRomFsProvider; -import libKonogonka.Tools.RomFs.RomFsEncryptedProvider; +import libKonogonka.Tools.RomFs.RomFsProvider; import libKonogonka.ctraes.AesCtrBufferedInputStream; import libKonogonka.ctraes.AesCtrDecryptSimple; import libKonogonka.exceptions.EmptySectionException; @@ -32,7 +32,6 @@ import org.apache.logging.log4j.Logger; import java.io.*; import java.nio.file.Files; import java.nio.file.Paths; -import java.util.LinkedList; public class NCAContent { private final static Logger log = LogManager.getLogger(NCAContent.class); @@ -43,7 +42,6 @@ public class NCAContent { private final NCAHeaderTableEntry ncaHeaderTableEntry; private final byte[] decryptedKey; - private LinkedList Pfs0SHA256hashes; private IPFS0Provider pfs0; private IRomFsProvider romfs; @@ -59,8 +57,6 @@ public class NCAContent { this.ncaFsHeader = ncaFsHeader; this.ncaHeaderTableEntry = ncaHeaderTableEntry; this.decryptedKey = decryptedKey; - System.out.println("NCAContent pfs0offsetPosition: "+ncaOffsetPosition); - Pfs0SHA256hashes = new LinkedList<>(); // If nothing to do if (ncaHeaderTableEntry.getMediaEndOffset() == 0) throw new EmptySectionException("Empty section"); @@ -91,7 +87,6 @@ public class NCAContent { ncaFsHeader.getSuperBlockPFS0(), ncaHeaderTableEntry.getMediaStartOffset(), ncaHeaderTableEntry.getMediaEndOffset()); - Pfs0SHA256hashes = pfs0.getPfs0SHA256hashes(); } private void proceedPFS0Encrypted() throws Exception{ @@ -103,16 +98,15 @@ public class NCAContent { decryptor, ncaHeaderTableEntry.getMediaStartOffset(), ncaHeaderTableEntry.getMediaEndOffset()); - Pfs0SHA256hashes = pfs0.getPfs0SHA256hashes(); } private void proceedRomFs() throws Exception{ switch (ncaFsHeader.getCryptoType()){ case 0x01: - proceedRomFsNotEncrypted(); // IF NO ENCRYPTION + proceedRomFsNotEncrypted(); break; case 0x03: - proceedRomFsEncrypted(); // If encrypted regular [ 0x03 ] + proceedRomFsEncrypted(); break; default: throw new Exception("Non-supported 'Crypto type'"); @@ -125,7 +119,7 @@ public class NCAContent { if (decryptedKey == null) throw new Exception("CryptoSection03: unable to proceed. No decrypted key provided."); - this.romfs = new RomFsEncryptedProvider( + this.romfs = new RomFsProvider( ncaFsHeader.getSuperBlockIVFC().getLvl6Offset(), file, ncaOffsetPosition, diff --git a/src/main/java/libKonogonka/Tools/NCA/NCAProvider.java b/src/main/java/libKonogonka/Tools/NCA/NCAProvider.java index 585a711..f515e92 100644 --- a/src/main/java/libKonogonka/Tools/NCA/NCAProvider.java +++ b/src/main/java/libKonogonka/Tools/NCA/NCAProvider.java @@ -29,8 +29,6 @@ import javax.crypto.Cipher; import javax.crypto.spec.SecretKeySpec; import java.io.*; import java.nio.charset.StandardCharsets; -import java.nio.file.Files; -import java.nio.file.Paths; import java.util.Arrays; import java.util.HashMap; diff --git a/src/main/java/libKonogonka/Tools/PFS0/PFS0Provider.java b/src/main/java/libKonogonka/Tools/PFS0/PFS0Provider.java index 484bef3..94f5eaa 100644 --- a/src/main/java/libKonogonka/Tools/PFS0/PFS0Provider.java +++ b/src/main/java/libKonogonka/Tools/PFS0/PFS0Provider.java @@ -33,8 +33,7 @@ import java.nio.file.Paths; import java.util.Arrays; import java.util.LinkedList; -import static libKonogonka.Converter.getLEint; -import static libKonogonka.Converter.getLElong; +import static libKonogonka.Converter.*; public class PFS0Provider implements IPFS0Provider{ private final static Logger log = LogManager.getLogger(PFS0Provider.class); @@ -295,11 +294,68 @@ public class PFS0Provider implements IPFS0Provider{ return true; } - //TODO: REMOVE + /** + * @deprecated + * */ @Override - public PipedInputStream getProviderSubFilePipedInpStream(String subFileName) throws Exception {return null;} + public PipedInputStream getProviderSubFilePipedInpStream(String subFileName) throws Exception { + for (int i = 0; i < pfs0subFiles.length; i++) { + if (pfs0subFiles[i].getName().equals(subFileName)) + return getProviderSubFilePipedInpStream(i); + } + throw new Exception("No file with such name exists: "+subFileName); + } + /** + * @deprecated + * */ @Override - public PipedInputStream getProviderSubFilePipedInpStream(int subFileNumber) throws Exception {return null;} + public PipedInputStream getProviderSubFilePipedInpStream(int subFileNumber) throws Exception { + PipedOutputStream streamOut = new PipedOutputStream(); + PipedInputStream streamInp = new PipedInputStream(streamOut); + + Thread workerThread = new Thread(() -> { + try { + PFS0subFile subFile = pfs0subFiles[subFileNumber]; + + if (encrypted) + createAesCtrEncryptedBufferedInputStream(); + else + createBufferedInputStream(); + + long subFileSize = subFile.getSize(); + + long toSkip = subFile.getOffset() + mediaStartOffset * 0x200 + rawBlockDataStart; + if (toSkip != stream.skip(toSkip)) + throw new Exception("Unable to skip offset: " + toSkip); + + int blockSize = 0x200; + if (subFileSize < 0x200) + blockSize = (int) subFileSize; + + long i = 0; + byte[] block = new byte[blockSize]; + + int actuallyRead; + while (true) { + if ((actuallyRead = stream.read(block)) != blockSize) + throw new Exception("Read failure. Block Size: " + blockSize + ", actuallyRead: " + actuallyRead); + streamOut.write(block); + i += blockSize; + if ((i + blockSize) > subFileSize) { + blockSize = (int) (subFileSize - i); + if (blockSize == 0) + break; + block = new byte[blockSize]; + } + } + } + catch (Exception e){ + log.error(e); + } + }); + workerThread.start(); + return streamInp; + } public LinkedList getPfs0SHA256hashes() { diff --git a/src/main/java/libKonogonka/Tools/RomFs/Level6Header.java b/src/main/java/libKonogonka/Tools/RomFs/Level6Header.java index 2a6d128..9f1b982 100644 --- a/src/main/java/libKonogonka/Tools/RomFs/Level6Header.java +++ b/src/main/java/libKonogonka/Tools/RomFs/Level6Header.java @@ -27,7 +27,7 @@ import org.apache.logging.log4j.Logger; * This class stores information contained in Level 6 Header of the RomFS image * ------------------------------------ * | Header Length (usually 0x50) | - * | Directory Hash Table Offset | Not used by this library | '<< 32' to get real offset: see implementation + * | Directory Hash Table Offset | Not used by this library * | Directory Hash Table Length | Not used by this library * | Directory Metadata Table Offset | * | Directory Metadata Table Length | @@ -42,7 +42,7 @@ public class Level6Header { private final static Logger log = LogManager.getLogger(Level6Header.class); private final long headerLength; - private long directoryHashTableOffset; + private final long directoryHashTableOffset; private final long directoryHashTableLength; private final long directoryMetadataTableOffset; private final long directoryMetadataTableLength; @@ -61,7 +61,7 @@ public class Level6Header { throw new Exception("Level 6 Header section is too small"); headerLength = getNext(); directoryHashTableOffset = getNext(); - directoryHashTableOffset <<= 32; + //directoryHashTableOffset <<= 32; directoryHashTableLength = getNext(); directoryMetadataTableOffset = getNext(); directoryMetadataTableLength = getNext(); @@ -73,7 +73,7 @@ public class Level6Header { } private long getNext(){ - final long result = Converter.getLEint(headerBytes, _cursor); + final long result = Converter.getLElongOfInt(headerBytes, _cursor); _cursor += 0x8; return result; } diff --git a/src/main/java/libKonogonka/Tools/RomFs/RomFsConstruct.java b/src/main/java/libKonogonka/Tools/RomFs/RomFsConstruct.java new file mode 100644 index 0000000..347e504 --- /dev/null +++ b/src/main/java/libKonogonka/Tools/RomFs/RomFsConstruct.java @@ -0,0 +1,160 @@ +/* + Copyright 2018-2022 Dmitry Isaenko + + This file is part of libKonogonka. + + libKonogonka is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + libKonogonka is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with libKonogonka. If not, see . + */ +package libKonogonka.Tools.RomFs; + +import libKonogonka.Converter; +import libKonogonka.ctraes.AesCtrBufferedInputStream; +import libKonogonka.ctraes.AesCtrDecryptSimple; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + +import java.io.BufferedInputStream; +import java.io.File; +import java.nio.file.Files; + +public class RomFsConstruct { + private final static Logger log = LogManager.getLogger(RomFsConstruct.class); + + private Level6Header header; + + private FileSystemEntry rootEntry; + private final BufferedInputStream stream; + private int headerSize; + private byte[] directoryMetadataTable; + private byte[] fileMetadataTable; + + + private final File file; + private long offsetPositionInFile; + private final long level6Offset; + + RomFsConstruct(File file, + long level6Offset) throws Exception{ + if (level6Offset < 0) + throw new Exception("Incorrect Level 6 Offset"); + this.file = file; + this.level6Offset = level6Offset; + this.stream = new BufferedInputStream(Files.newInputStream(file.toPath())); + constructEverything(); + } + + RomFsConstruct(File file, + long ncaOffset, // NCA offset position + long level6Offset, + AesCtrDecryptSimple decryptor, + long mediaStartOffset, + long mediaEndOffset + ) throws Exception { + if (level6Offset < 0) + throw new Exception("Incorrect Level 6 Offset"); + this.file = file; + this.level6Offset = level6Offset; + this.offsetPositionInFile = ncaOffset + (mediaStartOffset * 0x200); + // In 512-blocks + // In 512-blocks + this.stream = new AesCtrBufferedInputStream( + decryptor, + ncaOffset, + mediaStartOffset, + mediaEndOffset, + Files.newInputStream(file.toPath())); + constructEverything(); + } + + private void constructEverything() throws Exception{ + goToStartingPosition(); + + constructHeader(); + + directoryMetadataTableLengthCheck(); + directoryMetadataTableConstruct(); + + fileMetadataTableLengthCheck(); + fileMetadataTableConstruct(); + + constructRootFilesystemEntry(); + + stream.close(); + } + + private void goToStartingPosition() throws Exception{ + skipBytes(offsetPositionInFile + level6Offset); + } + + private void constructHeader() throws Exception{ + byte[] headerSizeBytes = detectHeaderSize(); + byte[] rawDataChunk = new byte[headerSize-0x8]; + + if (stream.read(rawDataChunk) != headerSize-0x8) + throw new Exception(String.format("Failed to read header (0x%x)", (headerSize-0x8))); + byte[] lv6headerBytes = new byte[headerSize]; + System.arraycopy(headerSizeBytes, 0, lv6headerBytes, 0, 0x8); + System.arraycopy(rawDataChunk, 0, lv6headerBytes, 0x8, headerSize-0x8); + this.header = new Level6Header(lv6headerBytes); + } + private byte[] detectHeaderSize() throws Exception{ + byte[] lv6HeaderSizeRaw = new byte[0x8]; + if (stream.read(lv6HeaderSizeRaw) != 0x8) + throw new Exception("Failed to read header size"); + headerSize = Converter.getLEint(lv6HeaderSizeRaw, 0); + return lv6HeaderSizeRaw; + } + + private void directoryMetadataTableLengthCheck() throws Exception{ + if (header.getDirectoryMetadataTableLength() < 0) + throw new Exception("Not supported: DirectoryMetadataTableLength < 0"); + } + private void directoryMetadataTableConstruct() throws Exception{ + skipBytes(header.getDirectoryMetadataTableOffset() - headerSize); + + directoryMetadataTable = new byte[(int) header.getDirectoryMetadataTableLength()]; + if (stream.read(directoryMetadataTable) != (int) header.getDirectoryMetadataTableLength()) + throw new Exception("Failed to read "+header.getDirectoryMetadataTableLength()); + } + + private void fileMetadataTableLengthCheck() throws Exception{ + if (header.getFileMetadataTableLength() < 0) + throw new Exception("Not supported: FileMetadataTableLength < 0"); + } + private void fileMetadataTableConstruct() throws Exception{ + skipBytes(header.getFileMetadataTableOffset() - header.getFileHashTableOffset()); + + fileMetadataTable = new byte[(int) header.getFileMetadataTableLength()]; + if (stream.read(fileMetadataTable) != (int) header.getFileMetadataTableLength()) + throw new Exception("Failed to read "+header.getFileMetadataTableLength()); + } + + private void constructRootFilesystemEntry() throws Exception{ + rootEntry = new FileSystemEntry(directoryMetadataTable, fileMetadataTable); + } + + private void skipBytes(long size) throws Exception{ + long mustSkip = size; + long skipped = 0; + while (mustSkip > 0){ + skipped += stream.skip(mustSkip); + mustSkip = size - skipped; + } + } + + Level6Header getHeader() { return header; } + FileSystemEntry getRootEntry(){ return rootEntry; } + byte[] getDirectoryMetadataTable() { return directoryMetadataTable; } + byte[] getFileMetadataTable() { return fileMetadataTable;} +} diff --git a/src/main/java/libKonogonka/Tools/RomFs/RomFsContentRetrieve.java b/src/main/java/libKonogonka/Tools/RomFs/RomFsContentRetrieve.java new file mode 100644 index 0000000..9fcd3f0 --- /dev/null +++ b/src/main/java/libKonogonka/Tools/RomFs/RomFsContentRetrieve.java @@ -0,0 +1,125 @@ +/* + Copyright 2018-2022 Dmitry Isaenko + + This file is part of libKonogonka. + + libKonogonka is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + libKonogonka is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with libKonogonka. If not, see . + */ +package libKonogonka.Tools.RomFs; + +import libKonogonka.ctraes.AesCtrBufferedInputStream; +import libKonogonka.ctraes.AesCtrDecryptSimple; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + +import java.io.*; +import java.nio.file.Files; + +public class RomFsContentRetrieve implements Runnable{ + private final static Logger log = LogManager.getLogger(RomFsContentRetrieve.class); + + private final PipedOutputStream streamOut; + private final long internalFileSize; + private final long startPosition; + private final BufferedInputStream bis; + + RomFsContentRetrieve(File parentFile, + PipedOutputStream streamOut, + long internalFileRealPosition, + long internalFileSize) throws Exception{ + this.streamOut = streamOut; + this.internalFileSize = internalFileSize; + + this.startPosition = internalFileRealPosition; + this.bis = new BufferedInputStream(Files.newInputStream(parentFile.toPath())); + } + + RomFsContentRetrieve(File parentFile, + PipedOutputStream streamOut, + AesCtrDecryptSimple decryptor, + long entryOffset, + long internalFileSize, + long headersFileDataOffset, //level6Header.getFileDataOffset() + long level6Offset, + long ncaOffsetPosition, + long mediaStartOffset, + long mediaEndOffset + ) throws Exception{ + log.fatal("Current implementation works incorrectly"); + this.streamOut = streamOut; + this.internalFileSize = internalFileSize; + + this.startPosition = entryOffset + mediaStartOffset*0x200 + headersFileDataOffset + level6Offset; + + this.bis = new AesCtrBufferedInputStream( + decryptor, + ncaOffsetPosition, + mediaStartOffset, + mediaEndOffset, + Files.newInputStream(parentFile.toPath()) + ); + } + + @Override + public void run() { + log.trace("Executing thread"); + try { + skipBytesTillBegining(); + + int readPiece = 8388608; + long readFrom = 0; + byte[] readBuffer; + + while (readFrom < internalFileSize) { + if (internalFileSize - readFrom < readPiece) + readPiece = Math.toIntExact(internalFileSize - readFrom); + readBuffer = new byte[readPiece]; + if (bis.read(readBuffer) != readPiece) { + log.error("getContent(): Unable to read requested size from file."); + return; + } + streamOut.write(readBuffer); + readFrom += readPiece; + } + } catch (Exception exception) { + log.error("RomFsProvider -> getContent(): Unable to provide stream", exception); + } + finally { + closeStreams(); + } + log.trace("Thread died"); + } + private void skipBytesTillBegining() throws Exception{ + long mustSkip = startPosition; + long skipped = 0; + while (mustSkip > 0){ + skipped += bis.skip(mustSkip); + mustSkip = startPosition - skipped; + } + } + private void closeStreams(){ + try { + streamOut.close(); + } + catch (IOException e){ + log.error("RomFsProvider -> getContent(): Unable to close 'StreamOut'"); + } + try { + bis.close(); + } + catch (IOException e){ + log.error("RomFsProvider -> getContent(): Unable to close 'StreamOut'"); + } + } +} diff --git a/src/main/java/libKonogonka/Tools/RomFs/RomFsDecryptedConstruct.java b/src/main/java/libKonogonka/Tools/RomFs/RomFsDecryptedConstruct.java deleted file mode 100644 index 50853dd..0000000 --- a/src/main/java/libKonogonka/Tools/RomFs/RomFsDecryptedConstruct.java +++ /dev/null @@ -1,121 +0,0 @@ -/* - Copyright 2018-2022 Dmitry Isaenko - - This file is part of libKonogonka. - - libKonogonka is free software: you can redistribute it and/or modify - it under the terms of the GNU General Public License as published by - the Free Software Foundation, either version 3 of the License, or - (at your option) any later version. - - libKonogonka is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU General Public License for more details. - - You should have received a copy of the GNU General Public License - along with libKonogonka. If not, see . - */ -package libKonogonka.Tools.RomFs; - -import libKonogonka.Converter; - -import java.io.BufferedInputStream; -import java.io.File; -import java.nio.file.Files; -/** - * Construct header for RomFs and obtain root fileSystemEntry (meta information) -* */ -class RomFsDecryptedConstruct { - private Level6Header header; - - private FileSystemEntry rootEntry; - private final BufferedInputStream fileBufferedInputStream; - private int headerSize; - private byte[] directoryMetadataTable; - private byte[] fileMetadataTable; - - RomFsDecryptedConstruct(File decryptedFsImageFile, long level6offset) throws Exception{ - if (level6offset < 0) - throw new Exception("Incorrect Level 6 Offset"); - - fileBufferedInputStream = new BufferedInputStream(Files.newInputStream(decryptedFsImageFile.toPath())); - fastForwardBySkippingBytes(level6offset); - - detectHeaderSize(); - constructHeader(); - - fastForwardBySkippingBytes(header.getDirectoryMetadataTableOffset() - headerSize); - - directoryMetadataTableLengthCheck(); - directoryMetadataTableConstruct(); - - fastForwardBySkippingBytes(header.getFileMetadataTableOffset() - header.getFileHashTableOffset()); - - fileMetadataTableLengthCheck(); - fileMetadataTableConstruct(); - - constructRootFilesystemEntry(); - - fileBufferedInputStream.close(); - } - private void detectHeaderSize() throws Exception{ - fileBufferedInputStream.mark(0x10); - byte[] lv6HeaderSizeRaw = new byte[0x8]; - if (fileBufferedInputStream.read(lv6HeaderSizeRaw) != 0x8) - throw new Exception("Failed to read header size"); - headerSize = Converter.getLEint(lv6HeaderSizeRaw, 0); - fileBufferedInputStream.reset(); - } - - private void constructHeader() throws Exception{ - byte[] rawDataChunk = new byte[headerSize]; - - if (fileBufferedInputStream.read(rawDataChunk) != headerSize) - throw new Exception(String.format("Failed to read header (0x%x)", headerSize)); - - this.header = new Level6Header(rawDataChunk); - } - - private void directoryMetadataTableLengthCheck() throws Exception{ - if (header.getDirectoryMetadataTableLength() < 0) - throw new Exception("Not supported operation."); - } - - private void directoryMetadataTableConstruct() throws Exception{ - directoryMetadataTable = new byte[(int) header.getDirectoryMetadataTableLength()]; - if (fileBufferedInputStream.read(directoryMetadataTable) != (int) header.getDirectoryMetadataTableLength()) - throw new Exception("Failed to read "+header.getDirectoryMetadataTableLength()); - } - - private void fileMetadataTableLengthCheck() throws Exception{ - if (header.getFileMetadataTableLength() < 0) - throw new Exception("Not supported operation."); - } - - private void fileMetadataTableConstruct() throws Exception{ - fileMetadataTable = new byte[(int) header.getFileMetadataTableLength()]; - - if (fileBufferedInputStream.read(fileMetadataTable) != (int) header.getFileMetadataTableLength()) - throw new Exception("Failed to read "+header.getFileMetadataTableLength()); - } - - private void constructRootFilesystemEntry() throws Exception{ - rootEntry = new FileSystemEntry(directoryMetadataTable, fileMetadataTable); - //rootEntry.printTreeForDebug(); - } - - private void fastForwardBySkippingBytes(long size) throws Exception{ - long mustSkip = size; - long skipped = 0; - while (mustSkip > 0){ - skipped += fileBufferedInputStream.skip(mustSkip); - mustSkip = size - skipped; - } - } - - Level6Header getHeader() { return header; } - FileSystemEntry getRootEntry(){ return rootEntry; } - byte[] getDirectoryMetadataTable() { return directoryMetadataTable; } - byte[] getFileMetadataTable() { return fileMetadataTable;} -} diff --git a/src/main/java/libKonogonka/Tools/RomFs/RomFsDecryptedContentRetrieve.java b/src/main/java/libKonogonka/Tools/RomFs/RomFsDecryptedContentRetrieve.java deleted file mode 100644 index 3d2e7b3..0000000 --- a/src/main/java/libKonogonka/Tools/RomFs/RomFsDecryptedContentRetrieve.java +++ /dev/null @@ -1,93 +0,0 @@ -/* - Copyright 2018-2022 Dmitry Isaenko - - This file is part of libKonogonka. - - libKonogonka is free software: you can redistribute it and/or modify - it under the terms of the GNU General Public License as published by - the Free Software Foundation, either version 3 of the License, or - (at your option) any later version. - - libKonogonka is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU General Public License for more details. - - You should have received a copy of the GNU General Public License - along with libKonogonka. If not, see . - */ -package libKonogonka.Tools.RomFs; - -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; - -import java.io.BufferedInputStream; -import java.io.File; -import java.io.IOException; -import java.io.PipedOutputStream; -import java.nio.file.Files; - -public class RomFsDecryptedContentRetrieve implements Runnable { - private final static Logger log = LogManager.getLogger(RomFsDecryptedContentRetrieve.class); - - private final File parentFile; - private final PipedOutputStream streamOut; - private final long internalFileRealPosition; - private final long internalFileSize; - - RomFsDecryptedContentRetrieve(File parentFile, - PipedOutputStream streamOut, - long internalFileRealPosition, - long internalFileSize){ - this.parentFile = parentFile; - this.streamOut = streamOut; - this.internalFileRealPosition = internalFileRealPosition; - this.internalFileSize = internalFileSize; - } - - @Override - public void run() { - log.trace("Executing thread"); - try (BufferedInputStream bis = new BufferedInputStream(Files.newInputStream(parentFile.toPath()))){ - fastForwardBySkippingBytes(bis, internalFileRealPosition); - - int readPice = 8388608; // 8mb NOTE: consider switching to 1mb 1048576 - long readFrom = 0; - byte[] readBuffer; - - while (readFrom < internalFileSize) { - if (internalFileSize - readFrom < readPice) - readPice = Math.toIntExact(internalFileSize - readFrom); // it's safe, I guarantee - readBuffer = new byte[readPice]; - if (bis.read(readBuffer) != readPice) { - log.error("getContent(): Unable to read requested size from file."); - return; - } - streamOut.write(readBuffer); - readFrom += readPice; - } - } catch (Exception exception) { - log.error("RomFsDecryptedProvider -> getContent(): Unable to provide stream", exception); - } - finally { - closeStreamOut(); - } - log.trace("Thread died"); - } - private void fastForwardBySkippingBytes(BufferedInputStream bis, long size) throws Exception{ - long mustSkip = size; - long skipped = 0; - while (mustSkip > 0){ - skipped += bis.skip(mustSkip); - mustSkip = size - skipped; - } - } - private void closeStreamOut(){ - try { - streamOut.close(); - } - catch (IOException e){ - log.error("RomFsDecryptedProvider -> getContent(): Unable to close 'StreamOut'"); - } - } -} diff --git a/src/main/java/libKonogonka/Tools/RomFs/RomFsDecryptedProvider.java b/src/main/java/libKonogonka/Tools/RomFs/RomFsDecryptedProvider.java deleted file mode 100644 index fecc077..0000000 --- a/src/main/java/libKonogonka/Tools/RomFs/RomFsDecryptedProvider.java +++ /dev/null @@ -1,76 +0,0 @@ -/* - * Copyright 2019-2022 Dmitry Isaenko - * - * This file is part of libKonogonka. - * - * libKonogonka is free software: you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * libKonogonka is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with libKonogonka. If not, see . - */ -package libKonogonka.Tools.RomFs; - -import libKonogonka.Tools.RomFs.view.DirectoryMetaTablePlainView; -import libKonogonka.Tools.RomFs.view.FileMetaTablePlainView; - -import java.io.*; - -public class RomFsDecryptedProvider implements IRomFsProvider{ - private final File file; - private final long level6Offset; - private final Level6Header level6Header; - private final FileSystemEntry rootEntry; - // Used only for debug - private final byte[] directoryMetadataTable; - private final byte[] fileMetadataTable; - - public RomFsDecryptedProvider(File decryptedFsImageFile, long level6offset) throws Exception{ - RomFsDecryptedConstruct construct = new RomFsDecryptedConstruct(decryptedFsImageFile, level6offset); - this.file = decryptedFsImageFile; - this.level6Offset = level6offset; - this.level6Header = construct.getHeader(); - this.rootEntry = construct.getRootEntry(); - - this.directoryMetadataTable = construct.getDirectoryMetadataTable(); - this.fileMetadataTable = construct.getFileMetadataTable(); - } - - @Override - public File getFile() { return file; } - @Override - public long getLevel6Offset() { return level6Offset; } - @Override - public Level6Header getHeader() { return level6Header; } - @Override - public FileSystemEntry getRootEntry() { return rootEntry; } - @Override - public PipedInputStream getContent(FileSystemEntry entry) throws Exception{ - if (entry.isDirectory()) - throw new Exception("Request of the binary stream for the folder entry is not supported (and doesn't make sense)."); - - PipedOutputStream streamOut = new PipedOutputStream(); - PipedInputStream streamIn = new PipedInputStream(streamOut); - long internalFileRealPosition = level6Offset + level6Header.getFileDataOffset() + entry.getOffset(); - long internalFileSize = entry.getSize(); - - Thread contentRetrievingThread = new Thread( - new RomFsDecryptedContentRetrieve(file, streamOut, internalFileRealPosition, internalFileSize)); - contentRetrievingThread.start(); - return streamIn; - } - @Override - public void printDebug(){ - level6Header.printDebugInfo(); - new DirectoryMetaTablePlainView(level6Header.getDirectoryMetadataTableLength(), directoryMetadataTable); - new FileMetaTablePlainView(level6Header.getFileMetadataTableLength(), fileMetadataTable); - rootEntry.printTreeForDebug(); - } -} diff --git a/src/main/java/libKonogonka/Tools/RomFs/RomFsEncryptedConstruct.java b/src/main/java/libKonogonka/Tools/RomFs/RomFsEncryptedConstruct.java deleted file mode 100644 index d90dea4..0000000 --- a/src/main/java/libKonogonka/Tools/RomFs/RomFsEncryptedConstruct.java +++ /dev/null @@ -1,180 +0,0 @@ -/* - Copyright 2018-2022 Dmitry Isaenko - - This file is part of libKonogonka. - - libKonogonka is free software: you can redistribute it and/or modify - it under the terms of the GNU General Public License as published by - the Free Software Foundation, either version 3 of the License, or - (at your option) any later version. - - libKonogonka is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU General Public License for more details. - - You should have received a copy of the GNU General Public License - along with libKonogonka. If not, see . - */ -package libKonogonka.Tools.RomFs; - -import libKonogonka.ctraes.AesCtrDecryptSimple; - -import java.io.File; -import java.io.RandomAccessFile; -import java.util.Arrays; - -public class RomFsEncryptedConstruct { - - private final long absoluteOffsetPosition; - private final long level6Offset; - - private final RandomAccessFile raf; - private final AesCtrDecryptSimple decryptor; - private Level6Header header; - private byte[] directoryMetadataTable; - private byte[] fileMetadataTable; - private FileSystemEntry rootEntry; - - RomFsEncryptedConstruct(File encryptedFsImageFile, - long romFsOffset, - long level6Offset, - AesCtrDecryptSimple decryptor, - long mediaStartOffset - ) throws Exception { - - if (level6Offset < 0) - throw new Exception("Incorrect Level 6 Offset"); - - this.raf = new RandomAccessFile(encryptedFsImageFile, "r"); - this.level6Offset = level6Offset; - this.absoluteOffsetPosition = romFsOffset + (mediaStartOffset * 0x200); - this.decryptor = decryptor; - raf.seek(absoluteOffsetPosition + level6Offset); - - //Go to Level 6 header position - decryptor.skipNext(level6Offset / 0x200); - - constructHeader(); - - directoryMetadataTableLengthCheck(); - directoryMetadataTableConstruct(); - - fileMetadataTableLengthCheck(); - fileMetadataTableConstruct(); - - constructRootFilesystemEntry(); - - raf.close(); - } - - private void constructHeader() throws Exception{ - // Decrypt data - byte[] encryptedBlock = new byte[0x200]; - byte[] decryptedBlock; - if (raf.read(encryptedBlock) == 0x200) - decryptedBlock = decryptor.decryptNext(encryptedBlock); - else - throw new Exception("Failed to read header header (0x200 - block)"); - this.header = new Level6Header(decryptedBlock); - } - - private void directoryMetadataTableLengthCheck() throws Exception{ - if (header.getDirectoryMetadataTableLength() < 0) - throw new Exception("Not supported: DirectoryMetadataTableLength < 0"); - } - private void directoryMetadataTableConstruct() throws Exception{ - directoryMetadataTable = readMetaTable(header.getDirectoryMetadataTableOffset(), - header.getDirectoryMetadataTableLength()); - } - - private void fileMetadataTableLengthCheck() throws Exception{ - if (header.getFileMetadataTableLength() < 0) - throw new Exception("Not supported: FileMetadataTableLength < 0"); - } - private void fileMetadataTableConstruct() throws Exception{ - fileMetadataTable = readMetaTable(header.getFileMetadataTableOffset(), - header.getFileMetadataTableLength()); - } - - private void constructRootFilesystemEntry() throws Exception{ - rootEntry = new FileSystemEntry(directoryMetadataTable, fileMetadataTable); - } - - private byte[] readMetaTable(long metaOffset, - long metaSize) throws Exception{ - byte[] encryptedBlock; - byte[] decryptedBlock; - byte[] metadataTable = new byte[(int) metaSize]; - //0 - decryptor.reset(); - - long startBlock = metaOffset / 0x200; - - decryptor.skipNext(level6Offset / 0x200 + startBlock); - - raf.seek(absoluteOffsetPosition + level6Offset + startBlock * 0x200); - - //1 - long ignoreBytes = metaOffset - startBlock * 0x200; - long currentPosition = 0; - - if (ignoreBytes > 0) { - encryptedBlock = new byte[0x200]; - if (raf.read(encryptedBlock) == 0x200) { - decryptedBlock = decryptor.decryptNext(encryptedBlock); - // If we have extra-small file that is less than a block and even more - if ((0x200 - ignoreBytes) > metaSize){ - metadataTable = Arrays.copyOfRange(decryptedBlock, (int)ignoreBytes, 0x200); - return metadataTable; - } - else { - System.arraycopy(decryptedBlock, (int) ignoreBytes, metadataTable, 0, 0x200 - (int) ignoreBytes); - currentPosition = 0x200 - ignoreBytes; - } - } - else { - throw new Exception("Unable to get 512 bytes from 1st bock for Directory/File Metadata Table"); - } - startBlock++; - } - long endBlock = (metaSize + ignoreBytes) / 0x200 + startBlock; // <- pointing to place where any data related to this media-block ends - - //2 - int extraData = (int) ((endBlock - startBlock)*0x200 - (metaSize + ignoreBytes)); - - if (extraData < 0) - endBlock--; - //3 - while ( startBlock < endBlock ) { - encryptedBlock = new byte[0x200]; - if (raf.read(encryptedBlock) == 0x200) { - decryptedBlock = decryptor.decryptNext(encryptedBlock); - System.arraycopy(decryptedBlock, 0, metadataTable, (int) currentPosition, 0x200); - } - else - throw new Exception("Unable to get 512 bytes from block for Directory/File Metadata Table"); - - startBlock++; - currentPosition += 0x200; - } - - //4 - if (extraData != 0){ // In case we didn't get what we want - encryptedBlock = new byte[0x200]; - if (raf.read(encryptedBlock) == 0x200) { - decryptedBlock = decryptor.decryptNext(encryptedBlock); - System.arraycopy(decryptedBlock, 0, metadataTable, (int) currentPosition, Math.abs(extraData)); - } - else - throw new Exception("Unable to get 512 bytes from block for Directory/File Metadata Table"); - } - - return metadataTable; - } - - Level6Header getHeader() { return header; } - FileSystemEntry getRootEntry(){ return rootEntry; } - byte[] getDirectoryMetadataTable() { return directoryMetadataTable; } - byte[] getFileMetadataTable() { return fileMetadataTable;} -} diff --git a/src/main/java/libKonogonka/Tools/RomFs/RomFsEncryptedContentRetrieve.java b/src/main/java/libKonogonka/Tools/RomFs/RomFsEncryptedContentRetrieve.java deleted file mode 100644 index 9f42af1..0000000 --- a/src/main/java/libKonogonka/Tools/RomFs/RomFsEncryptedContentRetrieve.java +++ /dev/null @@ -1,139 +0,0 @@ -/* - Copyright 2018-2022 Dmitry Isaenko - - This file is part of libKonogonka. - - libKonogonka is free software: you can redistribute it and/or modify - it under the terms of the GNU General Public License as published by - the Free Software Foundation, either version 3 of the License, or - (at your option) any later version. - - libKonogonka is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU General Public License for more details. - - You should have received a copy of the GNU General Public License - along with libKonogonka. If not, see . - */ -package libKonogonka.Tools.RomFs; - -import libKonogonka.ctraes.AesCtrDecryptSimple; -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; - -import java.io.File; -import java.io.PipedOutputStream; -import java.io.RandomAccessFile; - -public class RomFsEncryptedContentRetrieve implements Runnable{ - private final static Logger log = LogManager.getLogger(RomFsEncryptedContentRetrieve.class); - - private final File parentFile; - private final PipedOutputStream streamOut; - private final long absoluteOffsetPosition; - private final AesCtrDecryptSimple decryptor; - private final long internalFileOffset; - private final long internalFileSize; - private final long level6Offset; - private final long headersFileDataOffset; - - RomFsEncryptedContentRetrieve(File parentFile, - PipedOutputStream streamOut, - long absoluteOffsetPosition, - AesCtrDecryptSimple decryptor, - long internalFileOffset, - long internalFileSize, - long level6Offset, - long headersFileDataOffset - ){ - log.fatal("Current implementation works incorrectly"); - this.parentFile = parentFile; - this.absoluteOffsetPosition = absoluteOffsetPosition; - this.streamOut = streamOut; - this.decryptor = decryptor; - this.internalFileOffset = internalFileOffset; - this.internalFileSize = internalFileSize; - this.level6Offset = level6Offset; - this.headersFileDataOffset = headersFileDataOffset; - } - - @Override - public void run() { - log.trace("Executing thread"); - try { - byte[] encryptedBlock; - byte[] decryptedBlock; - - RandomAccessFile raf = new RandomAccessFile(parentFile, "r"); - - //0 - long startBlock = (internalFileOffset + headersFileDataOffset) / 0x200; - - decryptor.skipNext(level6Offset / 0x200 + startBlock); - - // long absoluteOffsetPosition = romFsOffsetPosition + (mediaStartOffset * 0x200); // calculated in constructor - - raf.seek(absoluteOffsetPosition + level6Offset + startBlock * 0x200); - - //1 - long ignoreBytes = (internalFileOffset + headersFileDataOffset) - startBlock * 0x200; - - if (ignoreBytes > 0) { - encryptedBlock = new byte[0x200]; - if (raf.read(encryptedBlock) == 0x200) { - decryptedBlock = decryptor.decryptNext(encryptedBlock); - // If we have extra-small file that is less than a block and even more - if ((0x200 - ignoreBytes) > internalFileSize){ - streamOut.write(decryptedBlock, (int)ignoreBytes, (int) internalFileSize); // safe cast - raf.close(); - streamOut.close(); - return; - } - else { - streamOut.write(decryptedBlock, (int) ignoreBytes, 0x200 - (int) ignoreBytes); - } - } - else { - throw new Exception("Unable to get 512 bytes from 1st bock"); - } - startBlock++; - } - long endBlock = (internalFileSize + ignoreBytes) / 0x200 + startBlock; // <- pointing to place where any data related to this media-block ends - - //2 - int extraData = (int) ((endBlock - startBlock)*0x200 - (internalFileSize + ignoreBytes)); - - if (extraData < 0) - endBlock--; - //3 - while ( startBlock < endBlock ) { - encryptedBlock = new byte[0x200]; - if (raf.read(encryptedBlock) == 0x200) { - decryptedBlock = decryptor.decryptNext(encryptedBlock); - streamOut.write(decryptedBlock); - } - else - throw new Exception("Unable to get 512 bytes from block"); - - startBlock++; - } - - //4 - if (extraData != 0){ // In case we didn't get what we want - encryptedBlock = new byte[0x200]; - if (raf.read(encryptedBlock) == 0x200) { - decryptedBlock = decryptor.decryptNext(encryptedBlock); - streamOut.write(decryptedBlock, 0, Math.abs(extraData)); - } - else - throw new Exception("Unable to get 512 bytes from block"); - } - raf.close(); - streamOut.close(); - } catch (Exception exception) { - log.error("Unable to provide stream", exception); - } - log.trace("Thread died"); - } -} diff --git a/src/main/java/libKonogonka/Tools/RomFs/RomFsEncryptedProvider.java b/src/main/java/libKonogonka/Tools/RomFs/RomFsProvider.java similarity index 54% rename from src/main/java/libKonogonka/Tools/RomFs/RomFsEncryptedProvider.java rename to src/main/java/libKonogonka/Tools/RomFs/RomFsProvider.java index 7d56987..afc18b3 100644 --- a/src/main/java/libKonogonka/Tools/RomFs/RomFsEncryptedProvider.java +++ b/src/main/java/libKonogonka/Tools/RomFs/RomFsProvider.java @@ -16,7 +16,6 @@ * You should have received a copy of the GNU General Public License * along with libKonogonka. If not, see . */ - package libKonogonka.Tools.RomFs; import libKonogonka.Tools.RomFs.view.DirectoryMetaTablePlainView; @@ -27,62 +26,70 @@ import java.io.File; import java.io.PipedInputStream; import java.io.PipedOutputStream; -public class RomFsEncryptedProvider implements IRomFsProvider{ +public class RomFsProvider implements IRomFsProvider{ private final File file; private final long level6Offset; private final Level6Header level6Header; private final FileSystemEntry rootEntry; - private final byte[] key; // Used @ createDecryptor only - private final byte[] sectionCTR; // Used @ createDecryptor only - private final long mediaStartOffset; // Used @ createDecryptor only - private final long absoluteOffsetPosition; - - //private long mediaEndOffset; // We know this, but actually never use - + private long ncaOffsetPosition; + private byte[] key; // Used @ createDecryptor only + private byte[] sectionCTR; // Used @ createDecryptor only + private long mediaStartOffset; // Used @ createDecryptor only + private long mediaEndOffset; // Used only for debug private final byte[] directoryMetadataTable; private final byte[] fileMetadataTable; - public RomFsEncryptedProvider(long level6Offset, - File encryptedFsImageFile, - long romFsOffsetPosition, - byte[] key, - byte[] sectionCTR, - long mediaStartOffset - ) throws Exception{ - this(level6Offset, encryptedFsImageFile, romFsOffsetPosition, key, sectionCTR, mediaStartOffset, -1); + private final boolean encryptedAesCtr; + + public RomFsProvider(File decryptedFsImageFile, long level6offset) throws Exception{ + RomFsConstruct construct = new RomFsConstruct(decryptedFsImageFile, level6offset); + this.file = decryptedFsImageFile; + this.level6Offset = level6offset; + this.level6Header = construct.getHeader(); + this.rootEntry = construct.getRootEntry(); + + this.directoryMetadataTable = construct.getDirectoryMetadataTable(); + this.fileMetadataTable = construct.getFileMetadataTable(); + + this.encryptedAesCtr = false; } - public RomFsEncryptedProvider(long level6Offset, - File encryptedFsImageFile, - long romFsOffsetPosition, - byte[] key, - byte[] sectionCTR, - long mediaStartOffset, - long mediaEndOffset + public RomFsProvider(long level6Offset, + File encryptedFsImageFile, + long ncaOffsetPosition, + byte[] key, + byte[] sectionCTR, + long mediaStartOffset, + long mediaEndOffset ) throws Exception{ this.key = key; this.sectionCTR = sectionCTR; this.mediaStartOffset = mediaStartOffset; + this.mediaEndOffset = mediaEndOffset; + this.ncaOffsetPosition = ncaOffsetPosition; - RomFsEncryptedConstruct construct = new RomFsEncryptedConstruct(encryptedFsImageFile, - romFsOffsetPosition, + RomFsConstruct construct = new RomFsConstruct(encryptedFsImageFile, + ncaOffsetPosition, level6Offset, createDecryptor(), - mediaStartOffset); + mediaStartOffset, + mediaEndOffset); this.file = encryptedFsImageFile; this.level6Offset = level6Offset; this.level6Header = construct.getHeader(); this.rootEntry = construct.getRootEntry(); - this.absoluteOffsetPosition = romFsOffsetPosition + (mediaStartOffset * 0x200); this.directoryMetadataTable = construct.getDirectoryMetadataTable(); this.fileMetadataTable = construct.getFileMetadataTable(); + + this.encryptedAesCtr = true; } private AesCtrDecryptSimple createDecryptor() throws Exception{ return new AesCtrDecryptSimple(key, sectionCTR, mediaStartOffset * 0x200); } + @Override public File getFile() { return file; } @Override @@ -92,7 +99,12 @@ public class RomFsEncryptedProvider implements IRomFsProvider{ @Override public FileSystemEntry getRootEntry() { return rootEntry; } @Override - public PipedInputStream getContent(FileSystemEntry entry) throws Exception{ + public PipedInputStream getContent(FileSystemEntry entry) throws Exception { + if (encryptedAesCtr) + return getContentAesCtrEncrypted(entry); + return getContentNonEncrypted(entry); + } + public PipedInputStream getContentAesCtrEncrypted(FileSystemEntry entry) throws Exception{ if (entry.isDirectory()) throw new Exception("Request of the binary stream for the folder entry is not supported (and doesn't make sense)."); @@ -101,19 +113,38 @@ public class RomFsEncryptedProvider implements IRomFsProvider{ long internalFileOffset = entry.getOffset(); long internalFileSize = entry.getSize(); - Thread contentRetrievingThread = new Thread(new RomFsEncryptedContentRetrieve( + Thread contentRetrievingThread = new Thread(new RomFsContentRetrieve( file, streamOut, - absoluteOffsetPosition, createDecryptor(), internalFileOffset, internalFileSize, + level6Header.getFileDataOffset(), level6Offset, - level6Header.getFileDataOffset() - )); + ncaOffsetPosition, + mediaStartOffset, + mediaEndOffset)); contentRetrievingThread.start(); return streamIn; } + public PipedInputStream getContentNonEncrypted(FileSystemEntry entry) throws Exception{ + if (entry.isDirectory()) + throw new Exception("Request of the binary stream for the folder entry is not supported (and doesn't make sense)."); + + PipedOutputStream streamOut = new PipedOutputStream(); + PipedInputStream streamIn = new PipedInputStream(streamOut); + long internalFileRealPosition = level6Offset + level6Header.getFileDataOffset() + entry.getOffset(); + long internalFileSize = entry.getSize(); + + Thread contentRetrievingThread = new Thread(new RomFsContentRetrieve( + file, + streamOut, + internalFileRealPosition, + internalFileSize)); + contentRetrievingThread.start(); + return streamIn; + } + @Override public void printDebug(){ level6Header.printDebugInfo(); diff --git a/src/main/java/libKonogonka/Tools/XCI/HFS0Provider.java b/src/main/java/libKonogonka/Tools/XCI/HFS0Provider.java index 6d5dd4d..9713a38 100644 --- a/src/main/java/libKonogonka/Tools/XCI/HFS0Provider.java +++ b/src/main/java/libKonogonka/Tools/XCI/HFS0Provider.java @@ -22,6 +22,7 @@ import libKonogonka.Tools.ISuperProvider; import java.io.*; import java.nio.charset.StandardCharsets; +import java.nio.file.Files; import java.util.Arrays; import static libKonogonka.Converter.*; @@ -30,59 +31,54 @@ import static libKonogonka.Converter.*; * HFS0 * */ public class HFS0Provider implements ISuperProvider { + private final String magic; + private final int filesCount; + private final byte[] padding; + private final int stringTableSize; + private final long rawFileDataStart; - private boolean magicHFS0; - private int filesCnt; - private boolean paddingHfs0; - private int stringTableSize; - private long rawFileDataStart; + private final HFS0File[] hfs0Files; - private HFS0File[] hfs0Files; - - private File file; + private final File file; HFS0Provider(long hfsOffsetPosition, RandomAccessFile raf, File file) throws Exception{ this.file = file; // Will be used @ getHfs0FilePipedInpStream. It's a bad implementation. byte[] hfs0bytes = new byte[16]; - try{ - raf.seek(hfsOffsetPosition); - if (raf.read(hfs0bytes) != 16){ - throw new Exception("Read HFS0 structure failure. Can't read first 16 bytes on requested offset."); - } + + raf.seek(hfsOffsetPosition); + if (raf.read(hfs0bytes) != 16){ + throw new Exception("Read HFS0 structure failure. Can't read first 16 bytes on requested offset."); } - catch (IOException ioe){ - throw new Exception("Read HFS0 structure failure. Can't read first 16 bytes on requested offset: "+ioe.getMessage()); - } - magicHFS0 = Arrays.equals(Arrays.copyOfRange(hfs0bytes, 0, 4),new byte[]{0x48, 0x46, 0x53, 0x30}); - filesCnt = getLEint(hfs0bytes, 0x4); - stringTableSize = getLEint(hfs0bytes, 8); - paddingHfs0 = Arrays.equals(Arrays.copyOfRange(hfs0bytes, 12, 16),new byte[4]); - hfs0Files = new HFS0File[filesCnt]; + this.magic = new String(hfs0bytes, 0x0, 0x4, StandardCharsets.US_ASCII); + this.filesCount = getLEint(hfs0bytes, 0x4); + this.stringTableSize = getLEint(hfs0bytes, 8); + this.padding = Arrays.copyOfRange(hfs0bytes, 12, 16); - // TODO: IF NOT EMPTY TABLE: + hfs0Files = new HFS0File[filesCount]; - long[] offsetHfs0files = new long[filesCnt]; - long[] sizeHfs0files = new long[filesCnt]; - int[] hashedRegionSizeHfs0Files = new int[filesCnt]; - boolean[] paddingHfs0Files = new boolean[filesCnt]; - byte[][] SHA256HashHfs0Files = new byte[filesCnt][]; - int[] strTableOffsets = new int[filesCnt]; + // TODO: IF NOT EMPTY TABLE: add validation + + long[] offsetSubFile = new long[filesCount]; + long[] sizeSubFile = new long[filesCount]; + int[] hashedRegionSubFile = new int[filesCount]; + boolean[] paddingSubFile = new boolean[filesCount]; + byte[][] SHA256HashSubFile = new byte[filesCount][]; + int[] stringTableOffsetSubFile = new int[filesCount]; try { // Populate meta information regarding each file inside (?) HFS0 byte[] metaInfoBytes = new byte[64]; - for (int i=0; i < filesCnt; i++){ - if (raf.read(metaInfoBytes) != 64) { + for (int i = 0; i < filesCount; i++){ + if (raf.read(metaInfoBytes) != 64) throw new Exception("Read HFS0 File Entry Table failure for file # "+i); - } - offsetHfs0files[i] = getLElong(metaInfoBytes, 0); - sizeHfs0files[i] = getLElong(metaInfoBytes, 8); - hashedRegionSizeHfs0Files[i] = getLEint(metaInfoBytes, 20); - paddingHfs0Files[i] = Arrays.equals(Arrays.copyOfRange(metaInfoBytes, 24, 32), new byte[8]); - SHA256HashHfs0Files[i] = Arrays.copyOfRange(metaInfoBytes, 32, 64); + offsetSubFile[i] = getLElong(metaInfoBytes, 0); + sizeSubFile[i] = getLElong(metaInfoBytes, 8); + hashedRegionSubFile[i] = getLEint(metaInfoBytes, 20); + paddingSubFile[i] = Arrays.equals(Arrays.copyOfRange(metaInfoBytes, 24, 32), new byte[8]); + SHA256HashSubFile[i] = Arrays.copyOfRange(metaInfoBytes, 32, 64); - strTableOffsets[i] = getLEint(metaInfoBytes, 16); + stringTableOffsetSubFile[i] = getLEint(metaInfoBytes, 16); } // Define location of actual data for this HFS0 rawFileDataStart = raf.getFilePointer()+stringTableSize; @@ -92,24 +88,24 @@ public class HFS0Provider implements ISuperProvider { if (raf.read(stringTbl) != stringTableSize){ throw new Exception("Read HFS0 String table failure. Can't read requested string table size ("+stringTableSize+")"); } - String[] namesHfs0files = new String[filesCnt]; + String[] namesSubFile = new String[filesCount]; // Parse string table - for (int i=0; i < filesCnt; i++){ + for (int i = 0; i < filesCount; i++){ int j = 0; - while (stringTbl[strTableOffsets[i]+j] != (byte)0x00) + while (stringTbl[stringTableOffsetSubFile[i]+j] != (byte)0x00) j++; - namesHfs0files[i] = new String(stringTbl, strTableOffsets[i], j, StandardCharsets.UTF_8); + namesSubFile[i] = new String(stringTbl, stringTableOffsetSubFile[i], j, StandardCharsets.UTF_8); } //---------------------------------------------------------------------------------------------------------- // Set files - for (int i=0; i < filesCnt; i++){ + for (int i = 0; i < filesCount; i++){ hfs0Files[i] = new HFS0File( - namesHfs0files[i], - offsetHfs0files[i], - sizeHfs0files[i], - hashedRegionSizeHfs0Files[i], - paddingHfs0Files[i], - SHA256HashHfs0Files[i] + namesSubFile[i], + offsetSubFile[i], + sizeSubFile[i], + hashedRegionSubFile[i], + paddingSubFile[i], + SHA256HashSubFile[i] ); } } @@ -118,15 +114,18 @@ public class HFS0Provider implements ISuperProvider { } } - public boolean isMagicHFS0() { return magicHFS0; } - public int getFilesCnt() { return filesCnt; } - public boolean isPaddingHfs0() { return paddingHfs0; } + public String getMagic() { return magic; } + public int getFilesCount() { return filesCount; } + public byte[] getPadding() { return padding; } public int getStringTableSize() { return stringTableSize; } @Override public long getRawFileDataStart() { return rawFileDataStart; } public HFS0File[] getHfs0Files() { return hfs0Files; } @Override public File getFile(){ return file; } + /** + * @deprecated + * */ @Override public PipedInputStream getProviderSubFilePipedInpStream(int subFileNumber) throws Exception{ PipedOutputStream streamOut = new PipedOutputStream(); @@ -140,13 +139,11 @@ public class HFS0Provider implements ISuperProvider { System.out.println("HFS0Provider -> getHfs0FilePipedInpStream(): Executing thread"); try{ long subFileRealPosition = rawFileDataStart + hfs0Files[subFileNumber].getOffset(); - BufferedInputStream bis = new BufferedInputStream(new FileInputStream(file)); - if (bis.skip(subFileRealPosition) != subFileRealPosition) { - System.out.println("HFS0Provider -> getHfs0FilePipedInpStream(): Unable to skip requested offset"); - return; - } + BufferedInputStream bis = new BufferedInputStream(Files.newInputStream(file.toPath())); + if (bis.skip(subFileRealPosition) != subFileRealPosition) + throw new Exception("HFS0Provider -> getHfs0FilePipedInpStream(): Unable to skip requested offset"); - int readPice = 8388608; // 8mb NOTE: consider switching to 1mb 1048576 + int readPice = 0x800000; // 8mb NOTE: consider switching to 1mb 1048576 long readFrom = 0; long realFileSize = hfs0Files[subFileNumber].getSize(); @@ -157,17 +154,16 @@ public class HFS0Provider implements ISuperProvider { if (realFileSize - readFrom < readPice) readPice = Math.toIntExact(realFileSize - readFrom); // it's safe, I guarantee readBuf = new byte[readPice]; - if (bis.read(readBuf) != readPice) { - System.out.println("HFS0Provider -> getHfs0FilePipedInpStream(): Unable to read requested size from file."); - return; - } + if (bis.read(readBuf) != readPice) + throw new Exception("HFS0Provider -> getHfs0FilePipedInpStream(): Unable to read requested size from file."); + streamOut.write(readBuf, 0, readPice); readFrom += readPice; } bis.close(); streamOut.close(); } - catch (IOException ioe){ + catch (Exception ioe){ System.out.println("HFS0Provider -> getHfs0FilePipedInpStream(): Unable to provide stream"); ioe.printStackTrace(); } @@ -176,20 +172,17 @@ public class HFS0Provider implements ISuperProvider { workerThread.start(); return streamIn; } - + //TODO @Override public boolean exportContent(String saveToLocation, String subFileName) throws Exception { - return false; + throw new Exception("Not implemented yet"); } - + //TODO @Override public boolean exportContent(String saveToLocation, int subFileNumber) throws Exception { - return false; + throw new Exception("Not implemented yet"); } - /** - * Sugar - * */ @Override public PipedInputStream getProviderSubFilePipedInpStream(String subFileName) throws Exception { for (int i = 0; i < hfs0Files.length; i++){ diff --git a/src/main/java/libKonogonka/Tools/XCI/XCIProvider.java b/src/main/java/libKonogonka/Tools/XCI/XCIProvider.java index bfd826b..cf25ad3 100644 --- a/src/main/java/libKonogonka/Tools/XCI/XCIProvider.java +++ b/src/main/java/libKonogonka/Tools/XCI/XCIProvider.java @@ -76,7 +76,7 @@ public class XCIProvider{ xciGamecardCert = new XCIGamecardCert(gamecardCertBytes); hfs0ProviderMain = new HFS0Provider(0xf000, raf, file); - if (hfs0ProviderMain.getFilesCnt() < 3){ + if (hfs0ProviderMain.getFilesCount() < 3){ raf.close(); throw new Exception("XCI Can't read Gamecard certificate bytes."); } @@ -106,7 +106,7 @@ public class XCIProvider{ throw new Exception("XCI Failed file analyze for ["+file.getName()+"]\n "+ioe.getMessage()); } } - /* Getters */ + /* API */ public XCIGamecardHeader getGCHeader(){ return this.xciGamecardHeader; } public XCIGamecardInfo getGCInfo(){ return this.xciGamecardInfo; } public XCIGamecardCert getGCCert(){ return this.xciGamecardCert; } diff --git a/src/main/java/libKonogonka/ctraes/AesCtrBufferedInputStream.java b/src/main/java/libKonogonka/ctraes/AesCtrBufferedInputStream.java index b4e8f8f..45eddf3 100644 --- a/src/main/java/libKonogonka/ctraes/AesCtrBufferedInputStream.java +++ b/src/main/java/libKonogonka/ctraes/AesCtrBufferedInputStream.java @@ -172,23 +172,14 @@ public class AesCtrBufferedInputStream extends BufferedInputStream { long leftovers = realCountOfBytesToSkip % 0x200; // most likely will be 0; TODO: a lot of tests long bytesToSkipTillRequiredBlock = realCountOfBytesToSkip - leftovers; - long skipped = super.skip(bytesToSkipTillRequiredBlock); - if (bytesToSkipTillRequiredBlock != skipped) - throw new IOException("Can't skip bytes. To skip: " + - bytesToSkipTillRequiredBlock + - ".\nActually skipped: " + skipped + - ".\nLeftovers inside encrypted section: " + leftovers); + skipLoop(bytesToSkipTillRequiredBlock); fillDecryptedCache(); pseudoPos += n; pointerInsideDecryptedSection = (int) leftovers; return n; } log.trace("4. Pointer Inside + End Position Outside Encrypted Section ("+pseudoPos+"-"+(pseudoPos+n)+")"); - long skipped = super.skip(realCountOfBytesToSkip); - if (realCountOfBytesToSkip != skipped) - throw new IOException("Can't skip bytes. To skip: " + - realCountOfBytesToSkip + - ".\nActually skipped: " + skipped); + skipLoop(realCountOfBytesToSkip); pseudoPos += n; pointerInsideDecryptedSection = 0; return n; @@ -217,12 +208,20 @@ public class AesCtrBufferedInputStream extends BufferedInputStream { return n; } log.trace("6. Not encrypted ("+pseudoPos+"-"+(pseudoPos+n)+")"); - long skipped = super.skip(n); + skipLoop(n); pseudoPos += n; pointerInsideDecryptedSection = 0; - return skipped; + return n; + } + private void skipLoop(long size) throws IOException{ + long mustSkip = size; + long skipped = 0; + while (mustSkip > 0){ + skipped += super.skip(mustSkip); + mustSkip = size - skipped; + log.trace("Skip loop: skipped: "+skipped+"\tmustSkip "+mustSkip); + } } - @Override public synchronized int read() throws IOException { byte[] b = new byte[1]; diff --git a/src/test/java/libKonogonka/RomFsDecrypted/RomFsDecryptedTest.java b/src/test/java/libKonogonka/RomFsDecrypted/RomFsDecryptedTest.java index 2b517fe..91d8ff2 100644 --- a/src/test/java/libKonogonka/RomFsDecrypted/RomFsDecryptedTest.java +++ b/src/test/java/libKonogonka/RomFsDecrypted/RomFsDecryptedTest.java @@ -21,7 +21,7 @@ package libKonogonka.RomFsDecrypted; import java.io.File; import java.nio.file.Path; -import libKonogonka.Tools.RomFs.RomFsDecryptedProvider; +import libKonogonka.Tools.RomFs.RomFsProvider; import org.junit.jupiter.api.Disabled; import org.junit.jupiter.api.DisplayName; import org.junit.jupiter.api.Test; @@ -36,7 +36,7 @@ public class RomFsDecryptedTest { private static final String decryptedFileAbsolutePath = "./FilesForTests/NCAContent_0 [lv6 147456].bin"; private File decryptedFile; long lv6offset; - RomFsDecryptedProvider provider; + RomFsProvider provider; @Disabled @DisplayName("RomFsDecryptedProvider: tests") @@ -55,7 +55,7 @@ public class RomFsDecryptedTest { lv6offset = Long.parseLong(decryptedFile.getName().replaceAll("(^.*lv6\\s)|(]\\.bin)", "")); } void makeProvider() throws Exception{ - provider = new RomFsDecryptedProvider(decryptedFile, lv6offset); + provider = new RomFsProvider(decryptedFile, lv6offset); } /* diff --git a/src/test/java/libKonogonka/RomFsDecrypted/RomFsEncryptedTest.java b/src/test/java/libKonogonka/RomFsDecrypted/RomFsEncryptedTest.java index a33ae68..a3e5771 100644 --- a/src/test/java/libKonogonka/RomFsDecrypted/RomFsEncryptedTest.java +++ b/src/test/java/libKonogonka/RomFsDecrypted/RomFsEncryptedTest.java @@ -117,7 +117,7 @@ public class RomFsEncryptedTest { exportFolderContent(entry, "/tmp/brandnew"); //---------------------------------------------------------------------- - exportFolderContentLegacy(entry, "/tmp/legacy"); + // exportFolderContentLegacy(entry, "/tmp/legacy"); } private void exportFolderContent(FileSystemEntry entry, String saveToLocation) throws Exception{ @@ -195,7 +195,7 @@ public class RomFsEncryptedTest { private void exportSingleFileLegacy(FileSystemEntry entry, String saveToLocation) throws Exception { File contentFile = new File(saveToLocation + entry.getName()); - BufferedOutputStream extractedFileBOS = new BufferedOutputStream(new FileOutputStream(contentFile)); + BufferedOutputStream extractedFileBOS = new BufferedOutputStream(Files.newOutputStream(contentFile.toPath())); PipedInputStream pis = ncaProvider.getNCAContentProvider(1).getRomfs().getContent(entry); byte[] readBuf = new byte[0x200]; // 8mb NOTE: consider switching to 1mb 1048576 diff --git a/src/test/java/libKonogonka/RomFsDecrypted/XciTest.java b/src/test/java/libKonogonka/RomFsDecrypted/XciTest.java new file mode 100644 index 0000000..479b907 --- /dev/null +++ b/src/test/java/libKonogonka/RomFsDecrypted/XciTest.java @@ -0,0 +1,66 @@ +/* + Copyright 2018-2022 Dmitry Isaenko + + This file is part of libKonogonka. + + libKonogonka is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + libKonogonka is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with libKonogonka. If not, see . + */ +package libKonogonka.RomFsDecrypted; + +import libKonogonka.Tools.XCI.XCIProvider; +import org.junit.jupiter.api.Disabled; +import org.junit.jupiter.api.DisplayName; +import org.junit.jupiter.api.Test; + +import java.io.BufferedReader; +import java.io.File; +import java.io.FileReader; + +// log.fatal("Configuration File Defined To Be :: "+System.getProperty("log4j.configurationFile")); + +public class XciTest { + private static final String xci_header_keyFileLocation = "./FilesForTests/xci_header_key.txt"; + private static final String decryptedFileAbsolutePath = "./FilesForTests/sample.xci"; + private File xciFile; + XCIProvider provider; + String xci_header_key; + + @Disabled + @DisplayName("RomFsDecryptedProvider: tests") + @Test + void romFsValidation() throws Exception{ + makeFile(); + getXciHeaderKey(); + makeProvider(); + } + + void getXciHeaderKey() throws Exception{ + BufferedReader br = new BufferedReader(new FileReader(xci_header_keyFileLocation)); + xci_header_key = br.readLine(); + br.close(); + + if (xci_header_key == null) + throw new Exception("Unable to retrieve xci_header_key"); + + xci_header_key = xci_header_key.trim(); + } + + void makeFile(){ + xciFile = new File(decryptedFileAbsolutePath); + } + + void makeProvider() throws Exception{ + provider = new XCIProvider(xciFile, xci_header_key); + } +} \ No newline at end of file