Skip to content

perf: speed up per-read validation hot loops (~25% less CPU) #2

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Draft
corneliusroemer-agent wants to merge 1 commit into
base: master
Choose a base branch
from
+66 −60
Draft

perf: speed up per-read validation hot loops (~25% less CPU) #2

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
56 changes: 44 additions & 12 deletions src/main/java/uk/ac/ebi/ena/readtools/v2/validator/InsdcReadsValidator.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -11,9 +11,7 @@
package uk.ac.ebi.ena.readtools.v2.validator;

import htsjdk.samtools.SAMException;
import java.util.HashSet;
import java.util.Iterator;
import java.util.Set;
import org.apache.commons.lang3.StringUtils;
import uk.ac.ebi.ena.readtools.v2.FileFormat;
import uk.ac.ebi.ena.readtools.v2.provider.ReadsProvider;
Expand All @@ -22,7 +20,22 @@

public class InsdcReadsValidator extends ReadsValidator {
public static final String IUPAC_CODES = "ACGTURYSWKMBDHVN.-";
private final Set<Character> iupacSet;
private static final String AUTCG_CODES = "AUTCG";

// ASCII lookup tables, case-folded at class-init, so the per-base hot loop needs no
// toUpperCase() allocation and no boxed HashSet lookup.
private static final boolean[] IUPAC_LOOKUP = buildLookup(IUPAC_CODES);
private static final boolean[] AUTCG_LOOKUP = buildLookup(AUTCG_CODES);

private static boolean[] buildLookup(String codes) {
boolean[] table = new boolean[128];
for (char c : codes.toCharArray()) {
table[Character.toUpperCase(c)] = true;
table[Character.toLowerCase(c)] = true;
}
return table;
}

private static final int MIN_QUALITY_SCORE = 30;

public static String ERROR_NULL_READS = "Reads cannot be null";
Expand All @@ -47,11 +60,6 @@ public class InsdcReadsValidator extends ReadsValidator {

public InsdcReadsValidator(long readCountLimit) {
super(readCountLimit);

iupacSet = new HashSet<>();
for (char c : IUPAC_CODES.toCharArray()) {
iupacSet.add(c);
}
}

public long getReadCount() {
Expand Down Expand Up @@ -127,15 +135,39 @@ public boolean validate(ReadsProviderFactory readsProviderFactory)
}

basesCount += effectiveBases.length();
for (char base : effectiveBases.toUpperCase().toCharArray()) {
if (iupacSet.contains(base)) {
if (base == 'A' || base == 'U' || base == 'T' || base == 'C' || base == 'G') {
autcgCount++;
// Fast path: pure-ASCII bases, checked against case-folded lookup tables with no
// allocation. Non-ASCII bases fall back to the original String.toUpperCase() path so
// that exotic case-folding (e.g. U+017F LATIN SMALL LETTER LONG S -> 'S') keeps
// behaving exactly as before.
int readAutcgCount = 0;
boolean nonAscii = false;
for (int i = 0, len = effectiveBases.length(); i < len; i++) {
char base = effectiveBases.charAt(i);
if (base >= 128) {
nonAscii = true;
break;
}
if (IUPAC_LOOKUP[base]) {
if (AUTCG_LOOKUP[base]) {
readAutcgCount++;
}
} else {
throw new ReadsValidationException(ERROR_NOT_IUPAC, readCount, effectiveBases);
}
}
if (nonAscii) {
readAutcgCount = 0;
for (char base : effectiveBases.toUpperCase().toCharArray()) {
if (IUPAC_CODES.indexOf(base) >= 0) {
if (AUTCG_CODES.indexOf(base) >= 0) {
readAutcgCount++;
}
} else {
throw new ReadsValidationException(ERROR_NOT_IUPAC, readCount, effectiveBases);
}
}
}
autcgCount += readAutcgCount;

int totalQuality = 0;
for (char q : effectiveQualityScores.toCharArray()) {
Expand Down
70 changes: 22 additions & 48 deletions src/main/java/uk/ac/ebi/ena/readtools/v2/validator/PairedFastqReadsValidator.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -70,57 +70,31 @@ protected void extraReadsValidation(ReadStyle readStyle, long readCount, FastqRe
addCount++;

if (pairingBloomWrapper != null && labels != null) {
// Name and index are derived from the same regex match, so match once and read both
// groups rather than running the same patterns twice per read.
String readName = read.getName();
if (readStyle == ReadStyle.CASAVA18) {
pairingBloomWrapper.add(getCasavaReadNameWithoutIndex(read.getName(), readCount));
labels.add(getCasavaReadIndex(read.getName(), readCount));
Matcher matcher = P_CASAVA_18_NAME.matcher(readName);
if (!matcher.matches()) {
throw new ReadsValidationException(
String.format("Line [%s] does not match %s regexp", readName, ReadStyle.CASAVA18),
readCount);
}
pairingBloomWrapper.add(matcher.group(1));
labels.add(matcher.group(3));
} else {
pairingBloomWrapper.add(getNonCasavaReadNameWithoutIndex(read.getName()));
labels.add(getNonCasavaReadIndex(read.getName()));
String nameWithoutIndex = readName;
String index = providerName;
if (!CASAVA_LIKE_EXCLUDE_REGEXP.matcher(readName).find()) {
Matcher m = SPLIT_REGEXP.matcher(readName);
if (m.find()) {
nameWithoutIndex = m.group(1);
index = m.group(2);
}
}
pairingBloomWrapper.add(nameWithoutIndex);
labels.add(index);
}
}
}

private String getCasavaReadNameWithoutIndex(String readName, long readIndex)
throws ReadsValidationException {
Matcher matcher = P_CASAVA_18_NAME.matcher(readName);
if (!matcher.matches()) {
throw new ReadsValidationException(
String.format("Line [%s] does not match %s regexp", readName, ReadStyle.CASAVA18),
readIndex);
}
return matcher.group(1);
}

private String getCasavaReadIndex(String readName, long readIndex)
throws ReadsValidationException {
Matcher matcher = P_CASAVA_18_NAME.matcher(readName);
if (!matcher.matches()) {
throw new ReadsValidationException(
String.format("Line [%s] does not match %s regexp", readName, ReadStyle.CASAVA18),
readIndex);
}
return matcher.group(3);
}

private String getNonCasavaReadNameWithoutIndex(String readName) {
Matcher casavaLikeMatcher = CASAVA_LIKE_EXCLUDE_REGEXP.matcher(readName);
if (!casavaLikeMatcher.find()) {
Matcher m = SPLIT_REGEXP.matcher(readName);
if (m.find()) {
return m.group(1);
}
}
return readName;
}

private String getNonCasavaReadIndex(String readName) {
Matcher casavaLikeMatcher = CASAVA_LIKE_EXCLUDE_REGEXP.matcher(readName);
if (!casavaLikeMatcher.find()) {
Matcher m = SPLIT_REGEXP.matcher(readName);
if (m.find()) {
return m.group(2);
}
}
return providerName;
}
}