summary logger: add expected-verdicts functionalityneels/summarylogger

2 files changed, 100 insertions, 2 deletions

diff --git a/libsummarylogger/SummaryLogger.cc b/libsummarylogger/SummaryLogger.cc
index 9cadb55a..ca306893 100644
--- a/libsummarylogger/SummaryLogger.cc
+++ b/libsummarylogger/SummaryLogger.cc
@@ -30,7 +30,7 @@ TestCase::TestCase()
 void TestCase::reset()
 {
 	verdict = Unbound;
-	expected_verdict = Skipped;
+	expected_verdict = NULL;
 	tc_name = "";
 	module_name = "";
 }
@@ -174,7 +174,23 @@ void SummaryLogger::log(const TitanLoggerApi::TitanLogEvent& event,
 }
 
 void TestCase::writeTestCase(FILE* file_stream_) const{
-	fprintf(file_stream_, "%s.%s: %s\n", module_name.data(), tc_name.data(), verdict);
+
+	if (!expected_verdict || expected_verdict == verdict) {
+		const char *verdict_str = verdict;
+		if (expected_verdict == Fail)
+			verdict_str = XFail;
+		/* We don't know what verdict to expect. Just write out results. */
+		fprintf(file_stream_, "%-5s", verdict_str);
+	} else {
+		/* There is an expected verdict read in from an expected-verdicts.txt.
+		 * Clearly indicate verdict changes. */
+		const char *expected_verdict_str = expected_verdict;
+		if (expected_verdict == Fail)
+			expected_verdict_str = XFail;
+
+		fprintf(file_stream_, "%s->%s", expected_verdict_str, verdict);
+	}
+	fprintf(file_stream_, " %s.%s\n", module_name.data(), tc_name.data());
 	fflush(file_stream_);
 }
 
@@ -189,6 +205,7 @@ void TestSuite::addTestCase(const TestCase& testcase) {
 }
 
 void TestSuite::write(FILE* file_stream_) {
+	readExpectedVerdicts();
 	for (TestCases::const_iterator it = testcases.begin(); it != testcases.end(); ++it) {
 		(*it)->writeTestCase(file_stream_);
 	}
@@ -224,3 +241,78 @@ void TestCase::setTCVerdict(const TitanLoggerApi::TitanLogEvent& event){
 		break;
 	}
 }
+
+void TestSuite::readExpectedVerdicts(const char *path)
+{
+	char *line = NULL;
+	char *saveptr = NULL;
+	size_t len = 0;
+	ssize_t read;
+	FILE *f;
+	if (!path)
+		path = "expected-verdicts.txt";
+	f = fopen(path, "r");
+	if (!f)
+		return;
+
+	while ((read = getline(&line, &len, f)) != -1) {
+		char *tokens[3] = {};
+		int i;
+
+		/* Expecting a line like written above:
+		 *   pass MSC_Tests.TC_foo
+		 *   [0]  [1]       [2]
+		 */
+		for (i = 0; i < 3; i++) {
+			tokens[i] = strtok_r(i ? NULL : line, " .\n", &saveptr);
+			if (!tokens[i]) {
+				i = -1;
+				break;
+			}
+		}
+
+		/* This line didn't have enough tokens. */
+		if (i != 3)
+			continue;
+
+		evaluateExpectedVerdict(tokens[1], tokens[2], tokens[0]);
+	}
+
+	fclose(f);
+
+}
+
+void TestSuite::evaluateExpectedVerdict(const char *suite_name, const char *test_name,
+					const char *verdict_name)
+{
+	for (TestCases::const_iterator it = testcases.begin(); it != testcases.end(); ++it) {
+		if ((*it)->evaluateExpectedVerdict(suite_name, test_name, verdict_name))
+			return;
+	}
+}
+
+bool TestCase::evaluateExpectedVerdict(const char *suite_name, const char *test_name,
+				       const char *verdict_name)
+{
+	if (strcmp(suite_name, this->module_name.data()))
+		return false;
+	if (strcmp(test_name, this->tc_name.data()))
+		return false;
+
+	if (!strcmp(verdict_name, Pass))
+		this->expected_verdict = Pass;
+	else if (!strcmp(verdict_name, Inconc))
+		this->expected_verdict = Inconc;
+	else if (!strcmp(verdict_name, Fail)
+		 || !strcmp(verdict_name, XFail))
+		this->expected_verdict = Fail;
+	else if (!strcmp(verdict_name, Error))
+		this->expected_verdict = Error;
+	else if (!strcmp(verdict_name, Unbound))
+		this->expected_verdict = Unbound;
+	else if (!strcmp(verdict_name, Skipped))
+		this->expected_verdict = Skipped;
+	else
+		return false;
+	return true;
+}
diff --git a/libsummarylogger/SummaryLogger.hh b/libsummarylogger/SummaryLogger.hh
index a98533a9..dadeaeae 100644
--- a/libsummarylogger/SummaryLogger.hh
+++ b/libsummarylogger/SummaryLogger.hh
@@ -29,6 +29,7 @@ struct TestCase {
 	static constexpr const char* Pass = "pass";
 	static constexpr const char* Inconc = "INCONCLUSIVE";
 	static constexpr const char* Fail = "FAIL";
+	static constexpr const char* XFail = "xfail";
 	static constexpr const char* Error = "ERROR";
 	static constexpr const char* Unbound = "UNBOUND";
 	static constexpr const char* Skipped = "skipped";
@@ -43,6 +44,8 @@ struct TestCase {
 	void writeTestCase(FILE* file_stream_) const;
 	void setTCVerdict(const TitanLoggerApi::TitanLogEvent& event);
 	void reset();
+	bool evaluateExpectedVerdict(const char *suite_name, const char *test_name,
+				     const char *verdict_name);
 };
 
 
@@ -62,6 +65,9 @@ struct TestSuite {
 	
 	void addTestCase(const TestCase& element);
 	void write(FILE* file_stream_);
+	void readExpectedVerdicts(const char *path=NULL);
+	void evaluateExpectedVerdict(const char *suite_name, const char *test_name,
+				     const char *verdict_name);
 };
 
 class SummaryLogger: public ILoggerPlugin