Skip to content

ETLManager:82 - Indexing version latest failed - #835 #408

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
mariusmarin-dev wants to merge 4 commits into
base: main
Choose a base branch
from
Open

ETLManager:82 - Indexing version latest failed - #835 #408

Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
6ab05fd
ETLManager:82 - Indexing version latest failed - #835
mariusmarin-dev Mar 23, 2026
151aa03
ETLManager:82 - Indexing version latest failed - #835
mariusmarin-dev Mar 23, 2026
1d8e630
ETLManager:82 - Indexing version latest failed - #835
mariusmarin-dev Mar 23, 2026
1e09a90
ETLManager:82 - Indexing version latest failed - #835
mariusmarin-dev Mar 24, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
126 changes: 78 additions & 48 deletions src/main/java/uk/ac/cam/cl/dtg/segue/etl/ContentIndexer.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -437,8 +437,11 @@ private Content augmentChildContent(final Content content, final String canonica

// hack to get cards to count as children:
if (content instanceof IsaacCardDeck) {
for (IsaacCard card : ((IsaacCardDeck) content).getCards()) {
this.augmentChildContent(card, canonicalSourceFile, newParentId, parentPublished);
List<IsaacCard> cards = ((IsaacCardDeck) content).getCards();
if (cards != null) {
for (IsaacCard card : cards) {
this.augmentChildContent(card, canonicalSourceFile, newParentId, parentPublished);
}
}
}

Expand Down Expand Up @@ -662,17 +665,19 @@ private synchronized void registerUnits(final IsaacNumericQuestion q, final Map<

HashMap<String, String> newUnits = Maps.newHashMap();

for (Choice c : q.getChoices()) {
if (c instanceof Quantity) {
Quantity quantity = (Quantity) c;
if (q.getChoices() != null) {
for (Choice c : q.getChoices()) {
if (c instanceof Quantity) {
Quantity quantity = (Quantity) c;

if (quantity.getUnits() != null && !quantity.getUnits().isEmpty()) {
String units = quantity.getUnits();
String cleanKey = units.replace("\t", "").replace("\n", "").replace(" ", "");
if (quantity.getUnits() != null && !quantity.getUnits().isEmpty()) {
String units = quantity.getUnits();
String cleanKey = units.replace("\t", "").replace("\n", "").replace(" ", "");

// May overwrite previous entry, doesn't matter as there is
// no mechanism by which to choose a winner
newUnits.put(cleanKey, units);
// May overwrite previous entry, doesn't matter as there is
// no mechanism by which to choose a winner
newUnits.put(cleanKey, units);
}
}
}
}
Expand Down Expand Up @@ -871,6 +876,16 @@ private void recordContentErrors(final String sha, final Map<String, Content> gi
log.info(String.format("Validation processing (%s) complete. There are %s files with content problems",
sanitiseInternalLogValue(sha), indexProblemCache.size()));

if (!indexProblemCache.isEmpty()) {
log.info("Found {} files with content problems:", indexProblemCache.size());
for (var entry : indexProblemCache.entrySet()) {
log.info("FILE: - {}", entry.getKey().getCanonicalSourceFile());
for (String problem : entry.getValue()) {
log.info("PROBLEM: - {}", problem);
}
}
}

if (indexProblemCache.isEmpty()) {
// Register a no-op style error to simplify application logic by ensuring there is always a content errors index
Content dummyContentRecord = new Content();
Expand Down Expand Up @@ -977,22 +992,24 @@ private void registerContentProblemsClozeQuestionChoicesHaveWrongNumberOFItems(
if (content instanceof IsaacClozeQuestion) {
IsaacClozeQuestion q = (IsaacClozeQuestion) content;
Integer numberItems = null;
for (Choice choice : q.getChoices()) {
if (choice instanceof ItemChoice) {
ItemChoice c = (ItemChoice) choice;
if (null == c.getItems() || c.getItems().isEmpty()) {
this.registerContentProblem(content, "Cloze Question: " + q.getId() + " has choice with missing items!",
indexProblemCache);
continue;
}
int items = c.getItems().size();
if (numberItems != null && items != numberItems) {
this.registerContentProblem(content,
"Cloze Question: " + q.getId() + " has choice with incorrect number of items!"
+ " (Expected " + numberItems + ", got " + items + "!)", indexProblemCache);
continue;
if (q.getChoices() != null) {
for (Choice choice : q.getChoices()) {
if (choice instanceof ItemChoice) {
ItemChoice c = (ItemChoice) choice;
if (null == c.getItems() || c.getItems().isEmpty()) {
this.registerContentProblem(content, "Cloze Question: " + q.getId() + " has choice with missing items!",
indexProblemCache);
continue;
}
int items = c.getItems().size();
if (numberItems != null && items != numberItems) {
this.registerContentProblem(content,
"Cloze Question: " + q.getId() + " has choice with incorrect number of items!"
+ " (Expected " + numberItems + ", got " + items + "!)", indexProblemCache);
continue;
}
numberItems = items;
}
numberItems = items;
}
}
}
Expand All @@ -1001,19 +1018,28 @@ private void registerContentProblemsClozeQuestionChoicesHaveWrongNumberOFItems(
private void registerContentProblemsSymbolicQuestionInvalidProperties(
final Content content, final Map<Content, List<String>> indexProblemCache) {
IsaacSymbolicQuestion question = (IsaacSymbolicQuestion) content;
for (String sym : question.getAvailableSymbols()) {
registerContentProblemQuestionSymbolContainsBackslash(content, indexProblemCache, question, sym);
}
for (Choice choice : question.getChoices()) {
if (choice instanceof Formula) {
Formula f = (Formula) choice;
if (f.getPythonExpression().contains("\\")) {
registerContentProblemQuestionFormulaContainsBackslash(content, indexProblemCache, question, choice);
} else if (f.getPythonExpression() == null || f.getPythonExpression().isEmpty()) {
registerContentProblemQuestionFormulaIsEmpty(content, indexProblemCache, question, choice);
if (question.getAvailableSymbols() != null) {
for (String sym : question.getAvailableSymbols()) {
registerContentProblemQuestionSymbolContainsBackslash(content, indexProblemCache, question, sym);
}
}
validateSymbolicQuestionChoices(content, indexProblemCache, question);
}

private void validateSymbolicQuestionChoices(final Content content,
final Map<Content, List<String>> indexProblemCache, final IsaacSymbolicQuestion question) {
if (question.getChoices() != null) {
for (Choice choice : question.getChoices()) {
if (choice instanceof Formula) {
Formula f = (Formula) choice;
if (f.getPythonExpression().contains("\\")) {
registerContentProblemQuestionFormulaContainsBackslash(content, indexProblemCache, question, choice);
} else if (f.getPythonExpression() == null || f.getPythonExpression().isEmpty()) {
registerContentProblemQuestionFormulaIsEmpty(content, indexProblemCache, question, choice);
}
} else {
registerContentProblemSymbolicQuestionChoiceIsNotFormula(content, indexProblemCache, question, choice);
}
} else {
registerContentProblemSymbolicQuestionChoiceIsNotFormula(content, indexProblemCache, question, choice);
}
}
}
Expand Down Expand Up @@ -1052,17 +1078,19 @@ private void registerContentProblemsNumericQuestionInvalidChoicesOrUnits(
final Content content, final Map<Content, List<String>> indexProblemCache) {
if (content instanceof IsaacNumericQuestion) {
IsaacNumericQuestion question = (IsaacNumericQuestion) content;
for (Choice choice : question.getChoices()) {
if (choice instanceof Quantity) {
Quantity quantity = (Quantity) choice;
if (question.getChoices() != null) {
for (Choice choice : question.getChoices()) {
if (choice instanceof Quantity) {
Quantity quantity = (Quantity) choice;

registerContentProblemCannotParseQuantityChoiceAsNumber(content, indexProblemCache, question, quantity);
registerContentProblemCannotParseQuantityChoiceAsNumber(content, indexProblemCache, question, quantity);

registerContentProblemUnnecessaryQuantityChoiceUnits(content, indexProblemCache, question, quantity);
registerContentProblemUnnecessaryQuantityChoiceUnits(content, indexProblemCache, question, quantity);


} else {
registerContentProblemNumericQuestionChoiceIsNotQuantity(content, indexProblemCache, question, choice);
} else {
registerContentProblemNumericQuestionChoiceIsNotQuantity(content, indexProblemCache, question, choice);
}
}
}
registerContentProblemConflictingUnitSettings(content, indexProblemCache, question);
Expand Down Expand Up @@ -1143,9 +1171,11 @@ private void registerContentProblemsChoiceQuestionMissingChoicesOrAnswer(
private void registerContentProblemChoiceQuestionMissingAnswer(
final Map<Content, List<String>> indexProblemCache, final ChoiceQuestion question) {
boolean correctOptionFound = false;
for (Choice choice : question.getChoices()) {
if (choice.isCorrect()) {
correctOptionFound = true;
if (question.getChoices() != null) {
for (Choice choice : question.getChoices()) {
if (choice.isCorrect()) {
correctOptionFound = true;
}
}
}
if (!correctOptionFound) {
Expand Down
Loading