Integer expectedValidError = 10; //так надо
expectedValidError *= -1;
BigDecimal abs = ((numberA.subtract(numberB)).divide(max)).abs();
BigDecimal powExpectedValidError = BigDecimal.valueOf(Math.pow(10, expectedValidError.doubleValue()));
boolean result = new BigDecimal(abs).compareTo(powExpectedValidError) <= 0;

java.lang.ArithmeticException: Non-terminating decimal expansion; no exact representable decimal result.
	at java.math.BigDecimal.divide(BigDecimal.java:1690)

String substractResult = v2.subtract(v1).toPlainString(); //получаю .0000000001

BigDecimal.valueOf(10).divide(BigDecimal.valueOf(9), 10, BigDecimal.ROUND_HALF_UP)