TokenStream sets back the cursor (to a value+1) when it shouldn't do that (only when EOF)

fix and testcases provided by @jcompagner; see mozilla#2367 for more

Author: rbri

rhino/src/main/java/org/mozilla/javascript/TokenStream.java

@@ -781,13 +781,9 @@ final int getToken() throws IOException {
                     }
                 }
 
-                // `ungetChar` will decrement the cursor, however the _actual_ tokenEnd should
-                // still be restored correctly after `getStringFromBuffer()` mutates it, so we save
-                // and restore it.
-                int savedTokenEnd = cursor;
-                ungetChar(c); // decrements cursor
+                if (c != EOF_CHAR)
+                    ungetChar(c); // decrements cursor, don't do this if the char is already EOF
                 String str = getStringFromBuffer(); // mutates tokenEnd to point to cursor
-                tokenEnd = savedTokenEnd; // restore tokenEnd
 
                 if (!containsEscape
                         || parser.compilerEnv.getLanguageVersion() >= Context.VERSION_ES6) {

rhino/src/test/java/org/mozilla/javascript/tests/ArrowFunctionTest.java

@@ -10,6 +10,7 @@
 import org.junit.jupiter.api.Test;
 import org.mozilla.javascript.Parser;
 import org.mozilla.javascript.ast.AstNode;
+import org.mozilla.javascript.ast.Block;
 import org.mozilla.javascript.ast.FunctionNode;
 import org.mozilla.javascript.ast.NodeVisitor;
 import org.mozilla.javascript.ast.ReturnStatement;
@@ -77,4 +78,60 @@ public void arrowFnEvalToString() {
         String source = "eval(\"()=>this\").toString()";
         Utils.assertWithAllModes("()=>this", source);
     }
+
+    @Test
+    public void testArrowRhinoWithLineEndingOnlyThis() {
+        FunctionNode arrowFn = parseAndExtractArrowFn("()=>this");
+
+        assertEquals(0, arrowFn.getPosition());
+        assertEquals(0, arrowFn.getAbsolutePosition());
+        assertEquals(8, arrowFn.getLength());
+
+        Block body = (Block) arrowFn.getBody();
+        assertEquals(4, body.getPosition());
+        assertEquals(4, body.getAbsolutePosition());
+        assertEquals(4, body.getLength());
+    }
+
+    @Test
+    public void testArrowRhinoWithLineEndingOnlyThisNl() {
+        FunctionNode arrowFn = parseAndExtractArrowFn("()=>this\n");
+
+        assertEquals(0, arrowFn.getPosition());
+        assertEquals(0, arrowFn.getAbsolutePosition());
+        assertEquals(8, arrowFn.getLength());
+
+        Block body = (Block) arrowFn.getBody();
+        assertEquals(4, body.getPosition());
+        assertEquals(4, body.getAbsolutePosition());
+        assertEquals(4, body.getLength());
+    }
+
+    @Test
+    public void testArrowRhinoWithLineEndingThisWithProperty() {
+        FunctionNode arrowFn = parseAndExtractArrowFn("()=>this.xs");
+
+        assertEquals(0, arrowFn.getPosition());
+        assertEquals(0, arrowFn.getAbsolutePosition());
+        assertEquals(11, arrowFn.getLength());
+
+        Block body = (Block) arrowFn.getBody();
+        assertEquals(4, body.getPosition());
+        assertEquals(4, body.getAbsolutePosition());
+        assertEquals(7, body.getLength());
+    }
+
+    @Test
+    public void testArrowRhinoWithLineEndingThisWithPropertyNl() {
+        FunctionNode arrowFn = parseAndExtractArrowFn("()=>this.xs\n");
+
+        assertEquals(0, arrowFn.getPosition());
+        assertEquals(0, arrowFn.getAbsolutePosition());
+        assertEquals(11, arrowFn.getLength());
+
+        Block body = (Block) arrowFn.getBody();
+        assertEquals(4, body.getPosition());
+        assertEquals(4, body.getAbsolutePosition());
+        assertEquals(7, body.getLength());
+    }
 }