package dokumenty.test; import java.io.File; import java.util.LinkedList; import java.util.List; import dokumenty.Element; import dokumenty.SerializatorElementow; import pl.wroc.pwr.file.ImageFolderScanner; import pl.wroc.pwr.image.converter.RGBtoGSConverter; import pl.wroc.pwr.image.io.GenericLoader; import pl.wroc.pwr.imagechannel.IImageChannel; public class ThresholdAccuracyTest { private enum PixelStatus {Omited, TruePositive, TrueNegative, FalsePositive, FalseNegative} private abstract class Rules {abstract PixelStatus Check(boolean Expected, boolean Result, boolean Threshold);} private class MariuszRules extends Rules { public PixelStatus Check(boolean Expected, boolean Result, boolean Threshold) { if (!Result && !Threshold) return PixelStatus.Omited; else if (!Expected && !Result) return PixelStatus.TrueNegative; else if (Expected && Result) return PixelStatus.TruePositive; else if (Expected && !Result) return PixelStatus.FalseNegative; else if (!Expected && Result) return PixelStatus.FalsePositive; else return PixelStatus.Omited; } } private class MichalRules extends Rules { public PixelStatus Check(boolean Expected, boolean Result, boolean Threshold) { if (!Threshold) return PixelStatus.Omited; else if (!Expected && !Result) return PixelStatus.TrueNegative; else if (Expected && Result) return PixelStatus.TruePositive; else if (Expected && !Result) return PixelStatus.FalseNegative; else if (!Expected && Result) return PixelStatus.FalsePositive; else return PixelStatus.Omited; } } private class FScore { private float Precision; private float Recall; public FScore(float Precision, float Recall) { this.Precision = Precision; this.Recall = Recall; } public FScore(float TP, float FP, float FN) { this.Precision = TP / (TP + FP); this.Recall = TP / (TP + FN); if (TP + FP == 0) this.Precision = 0.0f; } public float GetPrecision(){return Precision;} public float GetRecall(){return Recall;} public float GetFScore(float BetaSquare) { return (1 + BetaSquare) * ((Precision * Recall) / ((Precision * BetaSquare) + Recall)); } public float GetFScore() { return GetFScore(1.0f); } } private int Field(int[] box) { return (box[2] - box[0]) * (box[3] - box[1]); } private int[] Intersection(Element e1, Element e2) { int[] p1 = e1.pobierzPudelko(); int[] p2 = e2.pobierzPudelko(); int minX = Math.max(p1[0], p2[0]); int maxX = Math.min(p1[2], p2[2]); int minY = Math.max(p1[1], p2[1]); int maxY = Math.min(p1[3], p2[3]); if (minX < maxX && minY < maxY) return new int[]{minX, minY, maxX, maxY}; else return null; } private int[] Sum(Element e1, Element e2) { if (e1 == null) return e2.pobierzPudelko(); if (e2 == null) return e1.pobierzPudelko(); int[] p1 = e1.pobierzPudelko(); int[] p2 = e2.pobierzPudelko(); int minX = Math.min(p1[0], p2[0]); int maxX = Math.max(p1[2], p2[2]); int minY = Math.min(p1[1], p2[1]); int maxY = Math.max(p1[3], p2[3]); return new int[]{minX, minY, maxX, maxY}; } public void Test(String ImageLocationArgument, String RulesType, String[] Tags) throws IllegalArgumentException { float ColourThreshold = 0.9f; boolean OneToOne = false; Rules CurrentRules; if (RulesType.equalsIgnoreCase("MICHAL")) CurrentRules = new MichalRules(); else if (RulesType.equalsIgnoreCase("MARIUSZ")) CurrentRules = new MariuszRules(); else if (RulesType.equalsIgnoreCase("MICHAL-1TO1")) { CurrentRules = new MichalRules(); OneToOne = true; } else if (RulesType.equalsIgnoreCase("MARIUSZ-1TO1")) { CurrentRules = new MariuszRules(); OneToOne = true; } else throw new IllegalArgumentException(); String ImageLocation = ImageLocationArgument; //"C:\\NEKST\\Accuracy Test"; String ExpectedExtension = ".expected"; String ResultExtension = ".images"; String[] ExpectedTags = Tags;//{"textline", "paragraph", "pageheader", "pagefooter", "abstract", "header", "author", "title", "note"}; String[] ResultTags = Tags;//{"textline", "paragraph", "pageheader", "pagefooter", "abstract", "header", "author", "title", "note"}; List ExpectedTagsList = new LinkedList(); for (String CurrentString : ExpectedTags) ExpectedTagsList.add(CurrentString); List ResultTagsList = new LinkedList(); for (String CurrentString : ResultTags) ResultTagsList.add(CurrentString); System.out.println("Begining Accuracy Test..."); List Images = new ImageFolderScanner().apply(new File(ImageLocation)); SerializatorElementow ExpectedGetter = new SerializatorElementow(ExpectedExtension); SerializatorElementow ResultGetter = new SerializatorElementow(ResultExtension); float Precision = 0; float Recall = 0; int PrecisionNum = 0; int RecallNum = 0; System.out.println("Files found: " + Images.size()); for (File CurrentFile : Images) { long Begin = System.currentTimeMillis(); int TP = 0; int FP = 0; int FN = 0; System.out.println("\tProcessing file: " + CurrentFile); List ExpectedElements = ExpectedGetter.wczytaj(CurrentFile); List ResultElements = ResultGetter.wczytaj(CurrentFile); LinkedList FilteredExpectedElements = new LinkedList(); for (Element CurrentElement : ExpectedElements) if (ExpectedTagsList.contains(CurrentElement.pobierzTyp().toLowerCase())) FilteredExpectedElements.add(CurrentElement); LinkedList FilteredResultElements = new LinkedList(); for (Element CurrentElement : ResultElements) if (ResultTagsList.contains(CurrentElement.pobierzTyp().toLowerCase())) FilteredResultElements.add(CurrentElement); System.out.println("\t\tChosen expected elements: " + FilteredExpectedElements.size() + " of " + ExpectedElements.size()); System.out.println("\t\tChosen result elements: " + FilteredResultElements.size() + " of " + ResultElements.size()); FScore CurrentScore = new FScore(0, 0); if (FilteredExpectedElements.size() != 0 && FilteredResultElements.size() != 0) { IImageChannel CurrentImage = (new RGBtoGSConverter(new GenericLoader(CurrentFile).apply()).apply()).getGrayChannel(); if (OneToOne) { for (Element CurrentExpectedElement : FilteredExpectedElements) { Element BestResultElement = null; int IntersectionField = 0; for (Element CurrentResultElement : FilteredResultElements) { int[] CurrentIntersection = Intersection(CurrentExpectedElement, CurrentResultElement); if (CurrentIntersection != null) { int CurrentResult = Field(CurrentIntersection); if (CurrentResult > IntersectionField) { IntersectionField = CurrentResult; BestResultElement = CurrentResultElement; } } } int[] Box = Sum(CurrentExpectedElement, BestResultElement); for (int x = Math.max(0, Box[0]); x < Math.min(CurrentImage.getXSize(), Box[2] + 1); x++) { for (int y = Math.max(0, Box[1]); y < Math.min(CurrentImage.getYSize(), Box[3] + 1); y++) { PixelStatus CurrentPixel = CurrentRules.Check( BelongsTo(x, y, CurrentExpectedElement), BelongsTo(x, y, BestResultElement), CurrentImage.getValue(x, y) < ColourThreshold); if (CurrentPixel == PixelStatus.TruePositive) TP++; else if (CurrentPixel == PixelStatus.FalseNegative) FN++; else if (CurrentPixel == PixelStatus.FalsePositive) FP++; } } } } else { for (int x = 0; x < CurrentImage.getXSize(); x++) { for (int y = 0; y < CurrentImage.getYSize(); y++) { PixelStatus CurrentPixel = CurrentRules.Check( BelongsTo(x, y, FilteredExpectedElements), BelongsTo(x, y, FilteredResultElements), CurrentImage.getValue(x, y) < ColourThreshold); if (CurrentPixel == PixelStatus.TruePositive) TP++; else if (CurrentPixel == PixelStatus.FalseNegative) FN++; else if (CurrentPixel == PixelStatus.FalsePositive) FP++; } } } //System.out.println("\t\tTrue positives found: " + TP); //System.out.println("\t\tFlase negatives found: " + FN); //System.out.println("\t\tFalse positives found: " + FP); CurrentScore = new FScore(TP, FP, FN); Precision += CurrentScore.GetPrecision(); Recall += CurrentScore.GetRecall(); PrecisionNum++; RecallNum++; } else { if (FilteredExpectedElements.size() == 0 && FilteredResultElements.size() == 0) { System.out.println("\t\tNo expected or result elements found, skipping"); CurrentScore = new FScore(Float.NaN, Float.NaN); } else if (FilteredExpectedElements.size() == 0) { System.out.println("\t\tNo expected elements found"); CurrentScore = new FScore(Float.NaN, 0); Recall += CurrentScore.GetRecall(); RecallNum++; } else { System.out.println("\t\tNo result elements found"); CurrentScore = new FScore(0, Float.NaN); Precision += CurrentScore.GetPrecision(); PrecisionNum++; } } System.out.println("\t\tCurrent precision: " + CurrentScore.GetPrecision()); System.out.println("\t\tCurrent recall: " + CurrentScore.GetRecall()); System.out.println("\t\tCurrent F-score: " + CurrentScore.GetFScore()); long End = System.currentTimeMillis(); System.out.println("\t\tProcessing time: " + (End - Begin) + "ms"); } FScore FinalScore = new FScore(Precision / PrecisionNum, Recall / RecallNum); System.out.println("Total precision: " + FinalScore.GetPrecision()); System.out.println("Total recall: " + FinalScore.GetRecall()); System.out.println("Total F-score: " + FinalScore.GetFScore()); System.out.println(); System.out.println("Temporary score results:"); System.out.println(Precision + " " + PrecisionNum + " " + Recall + " " + RecallNum); } public static void main(String[] args) { System.out.println("TAC build: 08.06.2011"); try { if (args.length < 3) throw new IllegalArgumentException(); String[] tags = new String[args.length - 2]; for (int i = 0; i < tags.length; i++) tags[i] = args[i + 2].toLowerCase(); (new ThresholdAccuracyTest()).Test(args[0], args[1], tags); } catch (IllegalArgumentException Ex) { System.out.println("Illegal execution arguments!"); System.out.println("Should be: java -Xmx1024m -Xms512m -jar TAC.jar [database path] [MICHAL|MARIUSZ] [tag list]"); } } private boolean BelongsTo(int x, int y, LinkedList Elements) { for (Element CurrentElement : Elements) { int[] Box = CurrentElement.pobierzPudelko(); if (Box[0] <= x && Box[1] <= y && Box[2] >= x && Box[3] >= y) { Elements.remove(CurrentElement); Elements.addFirst(CurrentElement); return true; } } return false; } private boolean BelongsTo(int x, int y, Element CurrentElement) { if (CurrentElement == null) return false; int[] Box = CurrentElement.pobierzPudelko(); if (Box[0] <= x && Box[1] <= y && Box[2] >= x && Box[3] >= y) return true; else return false; } }