Skip to content
This repository has been archived by the owner on Jul 16, 2022. It is now read-only.

Commit

Permalink
Merge pull request #66 from eb4j/topic/miurahr/fix-wo-last-eol
Browse files Browse the repository at this point in the history
Optimize metadata loader
  • Loading branch information
miurahr authored Feb 28, 2022
2 parents 78c49a1 + a79241a commit d67b4e8
Show file tree
Hide file tree
Showing 7 changed files with 25 additions and 7 deletions.
7 changes: 4 additions & 3 deletions src/main/java/io/github/eb4j/dsl/DslDictionary.java
Original file line number Diff line number Diff line change
Expand Up @@ -43,8 +43,9 @@
public abstract class DslDictionary {
protected final DictionaryData<DslEntry> dictionaryData;
protected final DslDictionaryProperty prop;
protected final static String LINE_SEPARATOR = System.getProperty("line.separator");
protected final static Pattern DELIMITER_PATTERN = Pattern.compile("(\\r\\n|[\\n\\r\\u2028\\u2029\\u0085])+(\\s+)?");
protected static final String LINE_SEPARATOR = System.getProperty("line.separator");
protected static final Pattern DELIMITER_PATTERN = Pattern.compile(
"(\\r\\n|[\\n\\r\\u2028\\u2029\\u0085])+(\\s+)?");

protected DslDictionary(final DictionaryData<DslEntry> dictionaryData, final DslDictionaryProperty prop) {
this.dictionaryData = dictionaryData;
Expand Down Expand Up @@ -95,7 +96,7 @@ protected String getArticle(final DslEntry entry) throws IOException {
* @return article string.
* @throws IOException
*/
String getHeadWord(DslEntry entry) throws IOException {
String getHeadWord(final DslEntry entry) throws IOException {
return trimArticle(getRecord(entry.getHeaderOffset(), entry.getHeaderSize()));
}

Expand Down
2 changes: 2 additions & 0 deletions src/main/java/io/github/eb4j/dsl/DslDictionaryLoader.java
Original file line number Diff line number Diff line change
Expand Up @@ -285,6 +285,8 @@ private static Map<String, String> readMetadata(final Path path, final boolean i
break;
}
}
} else {
break;
}
}
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -40,10 +40,12 @@ public DictionaryDataBuilder() {
public DictionaryData<T> build(final List<DslIndex.Entry> entries) {
MapTrie<Object> mapPatriciaTrie = new MapPatriciaTrie<>();
for (DslIndex.Entry en: entries) {
doAdd(mapPatriciaTrie, en.getHeadWord(), en.getOffset(), en.getSize(), en.getHeaderOffset(), en.getHeaderSize());
doAdd(mapPatriciaTrie, en.getHeadWord(), en.getOffset(), en.getSize(),
en.getHeaderOffset(), en.getHeaderSize());
String lowerKey = en.getHeadWord().toLowerCase();
if (!en.getHeadWord().equals(lowerKey)) {
doAdd(mapPatriciaTrie, lowerKey, en.getOffset(), en.getSize(), en.getHeaderOffset(), en.getHeaderSize());
doAdd(mapPatriciaTrie, lowerKey, en.getOffset(), en.getSize(),
en.getHeaderOffset(), en.getHeaderSize());
}
}
return new DictionaryData<>(mapPatriciaTrie);
Expand Down
2 changes: 1 addition & 1 deletion src/main/java/io/github/eb4j/dsl/data/DslEntry.java
Original file line number Diff line number Diff line change
Expand Up @@ -49,7 +49,7 @@ public int getSize() {

@Override
@SuppressWarnings("NeedBraces")
public boolean equals(Object o) {
public boolean equals(final Object o) {
if (this == o) return true;
if (o == null || getClass() != o.getClass()) return false;

Expand Down
2 changes: 2 additions & 0 deletions src/main/java/io/github/eb4j/dsl/impl/EntriesLoaderImpl.java
Original file line number Diff line number Diff line change
Expand Up @@ -78,6 +78,7 @@ public void close() throws IOException {
}
}

@SuppressWarnings("innerassignment")
public List<DslIndex.Entry> load() throws IOException {
List<DslIndex.Entry> entries = new ArrayList<>();
long cardStart;
Expand Down Expand Up @@ -339,6 +340,7 @@ private int skipEmptyLine() throws IOException {
* @return -1 when EoF, otherwise a distance to eol from current position.
* @throws IOException when i/o error occurred.
*/
@SuppressWarnings("innerassignment")
private long eolSearch() throws IOException {
int b;
InputStream stream;
Expand Down
13 changes: 12 additions & 1 deletion src/test/java/io/github/eb4j/dsl/DslDictionaryTest.java
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,7 @@

class DslDictionaryTest {

protected final static String LINE_SEPARATOR = System.getProperty("line.separator");
protected static final String LINE_SEPARATOR = System.getProperty("line.separator");
private final URL resource = this.getClass().getResource("/utf16le_bom_crlf_el.dsl");

@Test
Expand Down Expand Up @@ -303,4 +303,15 @@ void loadUtf8_Comment() throws URISyntaxException, IOException {
Map.Entry<String, String> entry = results.getEntries(dumper).get(0);
assertEquals("Version: 0.0", entry.getValue());
}

@Test
void loadDictionaryUTF16LE_noel_wo_lasteof() throws URISyntaxException, IOException {
URL target = this.getClass().getResource("/utf16le_bom_crlf_noel_wo_lasteol.dsl");
DslDictionary dictionary = DslDictionary.loadDictionary(new File(target.toURI()));
DslResult res = dictionary.lookup("media");
DumpDslVisitor filter = new DumpDslVisitor();
Map.Entry entry = res.getEntries(filter).get(0);
assertEquals("[m][trn]this is media [s]image.jpg[/s] image and [video]video.ogv[/video][/trn][/m]",
entry.getValue());
}
}
Binary file not shown.

0 comments on commit d67b4e8

Please sign in to comment.