Save Authors and Genres to DB during parsing

This commit is contained in:
Dmitry Isaenko 2024-01-09 00:55:04 +03:00
parent 280dcd0fb0
commit 5a79d038eb
6 changed files with 125 additions and 114 deletions

View file

@ -1,11 +1,15 @@
package ru.redrise.marinesco.data; package ru.redrise.marinesco.data;
import java.util.Optional;
import org.springframework.data.repository.CrudRepository; import org.springframework.data.repository.CrudRepository;
import org.springframework.stereotype.Repository; import org.springframework.stereotype.Repository;
import ru.redrise.marinesco.library.Author; import ru.redrise.marinesco.library.Author;
@Repository @Repository
public interface AuthorRepository extends CrudRepository<Author, Long>{ public interface AuthorRepository extends CrudRepository<Author, Long>{
Optional<Author> findByAuthorName(String authorName);
} }

View file

@ -6,14 +6,12 @@ import jakarta.persistence.GeneratedValue;
import jakarta.persistence.GenerationType; import jakarta.persistence.GenerationType;
import jakarta.persistence.Id; import jakarta.persistence.Id;
import lombok.AccessLevel; import lombok.AccessLevel;
import lombok.AllArgsConstructor;
import lombok.Data; import lombok.Data;
import lombok.NoArgsConstructor; import lombok.NoArgsConstructor;
@Data @Data
@Entity @Entity
@NoArgsConstructor(access = AccessLevel.PRIVATE, force = true) @NoArgsConstructor(access = AccessLevel.PRIVATE, force = true)
@AllArgsConstructor
public class Author { public class Author {
// private static final long serialVersionUID = 1L; // private static final long serialVersionUID = 1L;

View file

@ -2,10 +2,13 @@ package ru.redrise.marinesco.library;
import jakarta.persistence.Entity; import jakarta.persistence.Entity;
import jakarta.persistence.Id; import jakarta.persistence.Id;
import lombok.AccessLevel;
import lombok.Data; import lombok.Data;
import lombok.NoArgsConstructor;
@Data @Data
@Entity @Entity
@NoArgsConstructor(access = AccessLevel.PRIVATE, force = true)
public class Genre { public class Genre {
@Id @Id
private String genreId; private String genreId;

View file

@ -14,6 +14,8 @@ import jakarta.persistence.Transient;
import lombok.Data; import lombok.Data;
import lombok.extern.slf4j.Slf4j; import lombok.extern.slf4j.Slf4j;
import ru.redrise.marinesco.RainbowDump; import ru.redrise.marinesco.RainbowDump;
import ru.redrise.marinesco.data.AuthorRepository;
import ru.redrise.marinesco.data.GenreRepository;
@Slf4j @Slf4j
@Entity @Entity
@ -39,65 +41,81 @@ public class InpEntry {
private String container; private String container;
@Transient @Transient
private int position; private int position = 0;
@Transient
private byte[] line;
public InpEntry(byte[] line, String container) throws Exception{ public InpEntry(byte[] line,
String container,
AuthorRepository authorRepository,
GenreRepository genreRepository) throws Exception {
// AUTHOR;GENRE;TITLE;SERIES;SERNO;FILE;SIZE;LIBID;DEL;EXT;DATE; // AUTHOR;GENRE;TITLE;SERIES;SERNO;FILE;SIZE;LIBID;DEL;EXT;DATE;
this.container = container; this.line = line;
this.container = container + ".zip";
this.authors = new ArrayList<>(); this.authors = new ArrayList<>();
this.genres = new ArrayList<>(); this.genres = new ArrayList<>();
parseAuthors(line); parseAuthors(authorRepository);
parseGenere(line); parseGenere(genreRepository);
this.title = parseNextString(line); this.title = parseNextString();
this.series = parseNextString(line); this.series = parseNextString();
this.serNo = parseNextString(line); this.serNo = parseNextString();
this.fsFileName = parseNextString(line); this.fsFileName = parseNextString();
this.fileSize = parseNextString(line); this.fileSize = parseNextString();
this.libId = parseNextString(line); this.libId = parseNextString();
this.deleted = parseNextString(line); this.deleted = parseNextString();
this.fileExtension = parseNextString(line); this.fileExtension = parseNextString();
this.addedDate = LocalDate.parse(parseNextString(line)); this.addedDate = LocalDate.parse(parseNextString());
/* /*
for (Author author : authors) * for (Author author : authors)
log.info(author.getAuthorName()); * log.info(author.getAuthorName());
for (Genre gen : genres) * for (Genre gen : genres)
log.info(gen.getGenreId()); * log.info(gen.getGenreId());
*
log.info(title); * log.info(title);
log.info(series); * log.info(series);
log.info(serNo); * log.info(serNo);
log.info(fsFileName); * log.info(fsFileName);
log.info(fileSize); * log.info(fileSize);
log.info(libId); * log.info(libId);
log.info(deleted); * log.info(deleted);
log.info(fileExtension); * log.info(fileExtension);
log.info(addedDate.toString()); * log.info(addedDate.toString());
*
log.info("-----------------"); * log.info("-----------------");
//*/ * //
*/
} }
private void parseAuthors(byte[] line) throws Exception{
for (int i = 0; i < line.length; i++){ private void parseAuthors(AuthorRepository authorRepository) throws Exception {
if (line[i] == 0x04){ for (; position < line.length; position++) {
splitAuthors(new String(line, 0, i, StandardCharsets.UTF_8)); if (line[position] == 0x04) {
position = i+1; String allAuthors = new String(line, 0, position, StandardCharsets.UTF_8);
for (String authorName : allAuthors.split(":")) {
Author author = authorRepository.findByAuthorName(authorName).orElse(new Author(authorName));
authors.add(authorRepository.save(author));
}
++position;
return; return;
} }
} }
RainbowDump.hexDumpUTF8(line);
throw new Exception("Invalid 'inp' file format (parse Authors)"); throw new Exception("Invalid 'inp' file format (parse Authors)");
} }
private void splitAuthors(String allAuthors){
for (String author : allAuthors.split(":")){
authors.add(new Author(author));
}
}
private void parseGenere(byte[] line) throws Exception{ private void parseGenere(GenreRepository genreRepository) throws Exception {
for (int i = position; i < line.length; i++) { for (int i = position; i < line.length; i++) {
if (line[i] == 0x04) { if (line[i] == 0x04) {
splitGenres(new String(line, 0, i, StandardCharsets.UTF_8)); String allGenres = new String(line, position, i - position, StandardCharsets.UTF_8);
for (String genreName : allGenres.split(":")) {
Genre genre = new Genre(genreName);
genres.add(genreRepository.save(genre));
}
position = i + 1; position = i + 1;
return; return;
} }
@ -105,13 +123,8 @@ public class InpEntry {
RainbowDump.hexDumpUTF8(line); RainbowDump.hexDumpUTF8(line);
throw new Exception("Invalid 'inp' file format (parse Genere)"); throw new Exception("Invalid 'inp' file format (parse Genere)");
} }
private void splitGenres(String allGenres){
for (String genre : allGenres.split(":")){
genres.add(new Genre(genre));
}
}
private String parseNextString(byte[] line) throws Exception{ private String parseNextString() throws Exception {
for (int i = position; i < line.length; i++) { for (int i = position; i < line.length; i++) {
if (line[i] == 0x04) { if (line[i] == 0x04) {
String resultingString = new String(line, position, i - position, StandardCharsets.UTF_8); String resultingString = new String(line, position, i - position, StandardCharsets.UTF_8);

View file

@ -1,36 +0,0 @@
package ru.redrise.marinesco.library;
import lombok.extern.slf4j.Slf4j;
@Slf4j
public class InpFileScanner {
private String name;
public InpFileScanner(byte[] content, String name) throws Exception{
this.name = name.substring(0, name.lastIndexOf('.'));
log.info("FILE RELATED "+this.name);
parseContent(content);
}
private void parseContent(byte[] content) throws Exception{
int lastIndex = 0;
for (int i = 0; i < content.length; i++){
if (content[i] == '\n'){
byte[] line = new byte[i-lastIndex];
System.arraycopy(content, lastIndex, line, 0, i-lastIndex-1);
new InpEntry(line, name);
//RainbowDump.hexDumpUTF8(line);
if (isNextCarriageReturn(i, content)){
i += 2;
lastIndex = i;
}
else
lastIndex = ++i;
}
}
}
private boolean isNextCarriageReturn(int i, byte[] content) {
return i + 1 < content.length && (content[i + 1] == '\r');
}
}

View file

@ -54,6 +54,7 @@ public class InpxScanner {
try (ZipInputStream zipInputStream = new ZipInputStream(new FileInputStream(inpxFile))) { try (ZipInputStream zipInputStream = new ZipInputStream(new FileInputStream(inpxFile))) {
ZipEntry zipEntry = zipInputStream.getNextEntry(); ZipEntry zipEntry = zipInputStream.getNextEntry();
while (zipEntry != null) { while (zipEntry != null) {
if (zipEntry.isDirectory()) { if (zipEntry.isDirectory()) {
zipEntry = zipInputStream.getNextEntry(); zipEntry = zipInputStream.getNextEntry();
@ -63,17 +64,19 @@ public class InpxScanner {
if (zipEntry.getName().toLowerCase().contains("collection.info")) if (zipEntry.getName().toLowerCase().contains("collection.info"))
libraryMetadata.setCollectionInfo(readPlainText(zipInputStream)); libraryMetadata.setCollectionInfo(readPlainText(zipInputStream));
if (zipEntry.getName().toLowerCase().contains("version.info")) else if (zipEntry.getName().toLowerCase().contains("version.info"))
libraryMetadata.setVersionInfo(readPlainText(zipInputStream)); libraryMetadata.setVersionInfo(readPlainText(zipInputStream));
if (zipEntry.getName().toLowerCase().endsWith(".inp")) { else if (zipEntry.getName().toLowerCase().endsWith(".inp")) {
/* //*
if (breaker) { if (breaker) {
zipEntry = zipInputStream.getNextEntry(); zipEntry = zipInputStream.getNextEntry();
continue; continue;
} }
breaker = true;// */ breaker = true;//
parseInp(zipInputStream, zipEntry.getSize(), zipEntry.getName()); //*/
byte[] content = inpToByteArray(zipInputStream, zipEntry.getSize());
parseInpContent(content, zipEntry.getName());
} }
zipEntry = zipInputStream.getNextEntry(); zipEntry = zipInputStream.getNextEntry();
@ -92,7 +95,7 @@ public class InpxScanner {
return stringBuilder.toString(); return stringBuilder.toString();
} }
private void parseInp(ZipInputStream stream, long fileSize, String fileName) throws Exception { private byte[] inpToByteArray(ZipInputStream stream, long fileSize) throws Exception {
ByteBuffer inpByteBuffer = ByteBuffer.allocate((int) fileSize); ByteBuffer inpByteBuffer = ByteBuffer.allocate((int) fileSize);
int blockSize = 0x200; int blockSize = 0x200;
if (fileSize < 0x200) if (fileSize < 0x200)
@ -113,9 +116,35 @@ public class InpxScanner {
block = new byte[blockSize]; block = new byte[blockSize];
} }
} }
// TODO : FIX! return inpByteBuffer.array();
//inpFileRepository.save(new InpFile(inpByteBuffer.array(), fileName)); }
new InpFileScanner(inpByteBuffer.array(), fileName);
private void parseInpContent(byte[] content, String name) throws Exception {
name = name.substring(0, name.lastIndexOf('.'));
log.info("FILE RELATED " + name);
int lastIndex = 0;
for (int i = 0; i < content.length; i++) {
if (content[i] == '\n') {
byte[] line = new byte[i - lastIndex];
System.arraycopy(content, lastIndex, line, 0, i - lastIndex - 1);
InpEntry book = new InpEntry(line, name, authorRepository, genreRepository);
//inpEntryRepository.save(book);
// RainbowDump.hexDumpUTF8(line);
if (isNextCarriageReturn(i, content)) {
i += 2;
lastIndex = i;
} else
lastIndex = ++i;
}
}
}
private boolean isNextCarriageReturn(int i, byte[] content) {
return i + 1 < content.length && (content[i + 1] == '\r');
} }
public String getFilesLocation() { public String getFilesLocation() {