Spock unclear assertion behavior - unit-testing

Question

As I know, one way to verify test result is to write expressions into the then section, that evaluates to boolean.
However lately I experienced a behavior which I don't understand. It seems like when one tries to verify something, that is in a block, then assertion only works with an explicit assert keyword.
Here is the example. I wrote a dummy if statement to have a block, but it's the same with a for loop or any control flow.
def "test fails as expected"() {
when: "result has some value"
def result = "someValue"
then: "result has the expected value"
result == "otherValue"
}
def "test passes, but shouldn't"() {
when: "result has some value"
def result = "someValue"
then: "result has the expected value"
if (true) {
result == "otherValue"
}
}
def "test fails as expected when using assert"() {
when: "result has some value"
def result = "someValue"
then: "result has the expected value"
if (true) {
assert result == "otherValue"
}
}
I find this behavior a bit misleading. Can someone explain why it works that way? Is this a bug or the usage is incorrect?

Answer 1

Following Spock documentation:
The when and then blocks always occur together. They describe a stimulus and the expected response. Whereas when blocks may contain arbitrary code, then blocks are restricted to conditions, exception conditions, interactions, and variable definitions. A feature method may contain multiple pairs of when-then blocks.
This explains why Spocks AST transformer does not see the following then block:
then:
if (true) {
result == "otherValue"
}
as a correct one and it does not transform it to SpockRuntime.verifyCondition() invocation.
If you compile your class (with static compilation enabled for better readability) and check the bytecode you will see something similar to this:
//
// Source code recreated from a .class file by IntelliJ IDEA
// (powered by Fernflower decompiler)
//
import groovy.lang.GroovyObject;
import org.codehaus.groovy.runtime.DefaultGroovyMethods;
import org.codehaus.groovy.runtime.ScriptBytecodeAdapter;
import org.spockframework.runtime.ErrorCollector;
import org.spockframework.runtime.SpockRuntime;
import org.spockframework.runtime.ValueRecorder;
import org.spockframework.runtime.model.BlockKind;
import org.spockframework.runtime.model.BlockMetadata;
import org.spockframework.runtime.model.FeatureMetadata;
import org.spockframework.runtime.model.SpecMetadata;
import spock.lang.Specification;
#SpecMetadata(
filename = "OtherSpec.groovy",
line = 4
)
public class OtherSpec extends Specification implements GroovyObject {
public OtherSpec() {
}
public Object test(String result) {
return true ? ScriptBytecodeAdapter.compareEqual(result, "otherValue") : null;
}
#FeatureMetadata(
line = 7,
name = "test fails as expected",
ordinal = 0,
blocks = {#BlockMetadata(
kind = BlockKind.WHEN,
texts = {"result has some value"}
), #BlockMetadata(
kind = BlockKind.THEN,
texts = {"result has the expected value"}
)},
parameterNames = {}
)
public void $spock_feature_0_0() {
ErrorCollector $spock_errorCollector = new ErrorCollector(false);
ValueRecorder $spock_valueRecorder = new ValueRecorder();
Object var10000;
try {
String result = "someValue";
try {
SpockRuntime.verifyCondition($spock_errorCollector, $spock_valueRecorder.reset(), "result == \"otherValue\"", Integer.valueOf(12), Integer.valueOf(9), (Object)null, $spock_valueRecorder.record($spock_valueRecorder.startRecordingValue(Integer.valueOf(2)), ScriptBytecodeAdapter.compareEqual($spock_valueRecorder.record($spock_valueRecorder.startRecordingValue(Integer.valueOf(0)), result), $spock_valueRecorder.record($spock_valueRecorder.startRecordingValue(Integer.valueOf(1)), "otherValue"))));
var10000 = null;
} catch (Throwable var13) {
SpockRuntime.conditionFailedWithException($spock_errorCollector, $spock_valueRecorder, "result == \"otherValue\"", Integer.valueOf(12), Integer.valueOf(9), (Object)null, var13);
var10000 = null;
} finally {
;
}
ScriptBytecodeAdapter.invokeMethod0(OtherSpec.class, ((OtherSpec)this).getSpecificationContext().getMockController(), (String)"leaveScope");
} finally {
$spock_errorCollector.validateCollectedErrors();
var10000 = null;
}
}
#FeatureMetadata(
line = 15,
name = "test passes, but shouldn't",
ordinal = 1,
blocks = {#BlockMetadata(
kind = BlockKind.WHEN,
texts = {"result has some value"}
), #BlockMetadata(
kind = BlockKind.THEN,
texts = {"result has the expected value"}
)},
parameterNames = {}
)
public void $spock_feature_0_1() {
String result = "someValue";
if (true) {
ScriptBytecodeAdapter.compareEqual(result, "otherValue");
}
ScriptBytecodeAdapter.invokeMethod0(OtherSpec.class, ((OtherSpec)this).getSpecificationContext().getMockController(), (String)"leaveScope");
}
#FeatureMetadata(
line = 25,
name = "test fails as expected when using assert",
ordinal = 2,
blocks = {#BlockMetadata(
kind = BlockKind.WHEN,
texts = {"result has some value"}
), #BlockMetadata(
kind = BlockKind.THEN,
texts = {"result has the expected value"}
)},
parameterNames = {}
)
public void $spock_feature_0_2() {
ErrorCollector $spock_errorCollector = new ErrorCollector(false);
ValueRecorder $spock_valueRecorder = new ValueRecorder();
Object var10000;
try {
String result = "someValue";
DefaultGroovyMethods.println(this, this.test("otherValue"));
var10000 = null;
if (true) {
try {
SpockRuntime.verifyCondition($spock_errorCollector, $spock_valueRecorder.reset(), "result == \"otherValue\"", Integer.valueOf(32), Integer.valueOf(20), (Object)null, $spock_valueRecorder.record($spock_valueRecorder.startRecordingValue(Integer.valueOf(2)), ScriptBytecodeAdapter.compareEqual($spock_valueRecorder.record($spock_valueRecorder.startRecordingValue(Integer.valueOf(0)), result), $spock_valueRecorder.record($spock_valueRecorder.startRecordingValue(Integer.valueOf(1)), "otherValue"))));
var10000 = null;
} catch (Throwable var13) {
SpockRuntime.conditionFailedWithException($spock_errorCollector, $spock_valueRecorder, "result == \"otherValue\"", Integer.valueOf(32), Integer.valueOf(20), (Object)null, var13);
var10000 = null;
} finally {
;
}
}
ScriptBytecodeAdapter.invokeMethod0(OtherSpec.class, ((OtherSpec)this).getSpecificationContext().getMockController(), (String)"leaveScope");
} finally {
$spock_errorCollector.validateCollectedErrors();
var10000 = null;
}
}
}
Now, if we analyze this code, we will find out that the following Spock test case:
def "test fails as expected"() {
when: "result has some value"
def result = "someValue"
then: "result has the expected value"
result == "otherValue"
}
compiles to something like this:
public void $spock_feature_0_0() {
ErrorCollector $spock_errorCollector = new ErrorCollector(false);
ValueRecorder $spock_valueRecorder = new ValueRecorder();
Object var10000;
try {
String result = "someValue";
try {
SpockRuntime.verifyCondition($spock_errorCollector, $spock_valueRecorder.reset(), "result == \"otherValue\"", Integer.valueOf(12), Integer.valueOf(9), (Object)null, $spock_valueRecorder.record($spock_valueRecorder.startRecordingValue(Integer.valueOf(2)), ScriptBytecodeAdapter.compareEqual($spock_valueRecorder.record($spock_valueRecorder.startRecordingValue(Integer.valueOf(0)), result), $spock_valueRecorder.record($spock_valueRecorder.startRecordingValue(Integer.valueOf(1)), "otherValue"))));
var10000 = null;
} catch (Throwable var13) {
SpockRuntime.conditionFailedWithException($spock_errorCollector, $spock_valueRecorder, "result == \"otherValue\"", Integer.valueOf(12), Integer.valueOf(9), (Object)null, var13);
var10000 = null;
} finally {
;
}
ScriptBytecodeAdapter.invokeMethod0(OtherSpec.class, ((OtherSpec)this).getSpecificationContext().getMockController(), (String)"leaveScope");
} finally {
$spock_errorCollector.validateCollectedErrors();
var10000 = null;
}
}
And the test case where you put assertion inside if-statement:
def "test passes, but shouldn't"() {
when: "result has some value"
def result = "someValue"
then: "result has the expected value"
if (true) {
result == "otherValue"
}
}
compiles to something like this:
public void $spock_feature_0_1() {
String result = "someValue";
if (true) {
ScriptBytecodeAdapter.compareEqual(result, "otherValue");
}
ScriptBytecodeAdapter.invokeMethod0(OtherSpec.class, ((OtherSpec)this).getSpecificationContext().getMockController(), (String)"leaveScope");
}
If you are interested in investigating the source code of this AST transformation you can start by analyzing:
org.spockframework.compiler.SpecRewriter.visitThenBlock() method
org.spockframework.compiler.DeepBlockRewriter.handleImplicitCondition() method
And for the last use case - adding assert to the if-statement block is the explicit instruction for Spock that it has to be transformed to verification condition invocation. That's why you see the bytecode that decompiles to something like this:
public void $spock_feature_0_2() {
ErrorCollector $spock_errorCollector = new ErrorCollector(false);
ValueRecorder $spock_valueRecorder = new ValueRecorder();
Object var10000;
try {
String result = "someValue";
DefaultGroovyMethods.println(this, this.test("otherValue"));
var10000 = null;
if (true) {
try {
SpockRuntime.verifyCondition($spock_errorCollector, $spock_valueRecorder.reset(), "result == \"otherValue\"", Integer.valueOf(32), Integer.valueOf(20), (Object)null, $spock_valueRecorder.record($spock_valueRecorder.startRecordingValue(Integer.valueOf(2)), ScriptBytecodeAdapter.compareEqual($spock_valueRecorder.record($spock_valueRecorder.startRecordingValue(Integer.valueOf(0)), result), $spock_valueRecorder.record($spock_valueRecorder.startRecordingValue(Integer.valueOf(1)), "otherValue"))));
var10000 = null;
} catch (Throwable var13) {
SpockRuntime.conditionFailedWithException($spock_errorCollector, $spock_valueRecorder, "result == \"otherValue\"", Integer.valueOf(32), Integer.valueOf(20), (Object)null, var13);
var10000 = null;
} finally {
;
}
}
ScriptBytecodeAdapter.invokeMethod0(OtherSpec.class, ((OtherSpec)this).getSpecificationContext().getMockController(), (String)"leaveScope");
} finally {
$spock_errorCollector.validateCollectedErrors();
var10000 = null;
}
}
Notice that if (true) { /*...*/ } is still present because AST transformer still ignores transforming it, but the condition:
assert result == "otherValue"
was visited and accepted by the AST transformer, and replaced by:
SpockRuntime.verifyCondition($spock_errorCollector, $spock_valueRecorder.reset(), "result == \"otherValue\"", Integer.valueOf(32), Integer.valueOf(20), (Object)null, $spock_valueRecorder.record($spock_valueRecorder.startRecordingValue(Integer.valueOf(2)), ScriptBytecodeAdapter.compareEqual($spock_valueRecorder.record($spock_valueRecorder.startRecordingValue(Integer.valueOf(0)), result), $spock_valueRecorder.record($spock_valueRecorder.startRecordingValue(Integer.valueOf(1)), "otherValue"))));