Merge pull request #481 from headius/jruby_updates

JRuby updates and fixes

2 files changed, 142 insertions, 52 deletions

diff --git a/ext/java/org/jruby/ext/psych/PsychEmitter.java b/ext/java/org/jruby/ext/psych/PsychEmitter.java
index 1372c2f..be6e388 100644
--- a/ext/java/org/jruby/ext/psych/PsychEmitter.java
+++ b/ext/java/org/jruby/ext/psych/PsychEmitter.java
@@ -29,13 +29,16 @@ package org.jruby.ext.psych;
 
 import java.io.IOException;
 import java.io.OutputStreamWriter;
+import java.io.Writer;
 import java.nio.charset.Charset;
 import java.util.HashMap;
 import java.util.Map;
 
 import org.jcodings.Encoding;
+import org.jcodings.specific.UTF8Encoding;
 import org.jruby.Ruby;
 import org.jruby.RubyArray;
+import org.jruby.RubyBoolean;
 import org.jruby.RubyClass;
 import org.jruby.RubyFixnum;
 import org.jruby.RubyModule;
@@ -46,6 +49,8 @@ import org.jruby.runtime.ObjectAllocator;
 import org.jruby.runtime.ThreadContext;
 import org.jruby.runtime.builtin.IRubyObject;
 import org.jruby.util.IOOutputStream;
+import org.jruby.util.TypeConverter;
+import org.jruby.util.io.EncodingUtils;
 import org.yaml.snakeyaml.DumperOptions;
 import org.yaml.snakeyaml.emitter.Emitter;
 import org.yaml.snakeyaml.emitter.EmitterException;
@@ -110,9 +115,7 @@ public class PsychEmitter extends RubyObject {
 
     @JRubyMethod
     public IRubyObject start_stream(ThreadContext context, IRubyObject encoding) {
-        if (!(encoding instanceof RubyFixnum)) {
-            throw context.runtime.newTypeError(encoding, context.runtime.getFixnum());
-        }
+        TypeConverter.checkType(context, encoding, context.runtime.getFixnum());
 
         initEmitter(context, encoding);
 
@@ -136,6 +139,8 @@ public class PsychEmitter extends RubyObject {
         boolean implicitBool = implicit.isTrue();
         Map<String, String> tagsMap = null;
 
+        TypeConverter.checkType(context, _version, context.runtime.getArray());
+
         RubyArray versionAry = _version.convertToArray();
         if (versionAry.size() == 2) {
             int versionInt0 = (int)versionAry.eltInternal(0).convertToInteger().getLongValue();
@@ -153,19 +158,23 @@ public class PsychEmitter extends RubyObject {
             }
         }
 
-        RubyArray tagsAry = tags.convertToArray();
-        if (tagsAry.size() > 0) {
-            tagsMap = new HashMap<String, String>(tagsAry.size());
-            for (int i = 0; i < tagsAry.size(); i++) {
-                RubyArray tagsTuple = tagsAry.eltInternal(i).convertToArray();
-                if (tagsTuple.size() != 2) {
-                    throw context.runtime.newRuntimeError("tags tuple must be of length 2");
+        if (!tags.isNil()) {
+            TypeConverter.checkType(context, tags, context.runtime.getArray());
+
+            RubyArray tagsAry = tags.convertToArray();
+            if (tagsAry.size() > 0) {
+                tagsMap = new HashMap<>(tagsAry.size());
+                for (int i = 0; i < tagsAry.size(); i++) {
+                    RubyArray tagsTuple = tagsAry.eltInternal(i).convertToArray();
+                    if (tagsTuple.size() != 2) {
+                        throw context.runtime.newRuntimeError("tags tuple must be of length 2");
+                    }
+                    IRubyObject key = tagsTuple.eltInternal(0);
+                    IRubyObject value = tagsTuple.eltInternal(1);
+                    tagsMap.put(
+                            key.asJavaString(),
+                            value.asJavaString());
                 }
-                IRubyObject key = tagsTuple.eltInternal(0);
-                IRubyObject value = tagsTuple.eltInternal(1);
-                tagsMap.put(
-                        key.asJavaString(),
-                        value.asJavaString());
             }
         }
 
@@ -189,21 +198,29 @@ public class PsychEmitter extends RubyObject {
         IRubyObject plain = args[3];
         IRubyObject quoted = args[4];
         IRubyObject style = args[5];
-        
-        if (!(value instanceof RubyString)) {
-            throw context.runtime.newTypeError(value, context.runtime.getString());
-        }
+
+        RubyClass stringClass = context.runtime.getString();
+
+        TypeConverter.checkType(context, value, stringClass);
+
+        RubyString valueStr = (RubyString) value;
+
+        valueStr = EncodingUtils.strConvEnc(context, valueStr, valueStr.getEncoding(), UTF8Encoding.INSTANCE);
+
+        RubyString anchorStr = exportToUTF8(context, anchor, stringClass);
+        RubyString tagStr = exportToUTF8(context, tag, stringClass);
 
         ScalarEvent event = new ScalarEvent(
-                anchor.isNil() ? null : anchor.asJavaString(),
-                tag.isNil() ? null : tag.asJavaString(),
-                new ImplicitTuple(plain.isTrue(),
-                quoted.isTrue()),
-                value.asJavaString(),
+                anchorStr == null ? null : anchorStr.asJavaString(),
+                tagStr == null ? null : tagStr.asJavaString(),
+                new ImplicitTuple(plain.isTrue(), quoted.isTrue()),
+                valueStr.asJavaString(),
                 NULL_MARK,
                 NULL_MARK,
                 SCALAR_STYLES[style.convertToInteger().getIntValue()]);
+
         emit(context, event);
+
         return this;
     }
 
@@ -214,11 +231,14 @@ public class PsychEmitter extends RubyObject {
         IRubyObject implicit = args[2];
         IRubyObject style = args[3];
 
-        final int SEQUENCE_BLOCK = 1; // see psych/nodes/sequence.rb
+        RubyClass stringClass = context.runtime.getString();
+
+        RubyString anchorStr = exportToUTF8(context, anchor, stringClass);
+        RubyString tagStr = exportToUTF8(context, tag, stringClass);
 
         SequenceStartEvent event = new SequenceStartEvent(
-                anchor.isNil() ? null : anchor.asJavaString(),
-                tag.isNil() ? null : tag.asJavaString(),
+                anchorStr == null ? null : anchorStr.asJavaString(),
+                tagStr == null ? null : tagStr.asJavaString(),
                 implicit.isTrue(),
                 NULL_MARK,
                 NULL_MARK,
@@ -241,16 +261,21 @@ public class PsychEmitter extends RubyObject {
         IRubyObject implicit = args[2];
         IRubyObject style = args[3];
 
-        final int MAPPING_BLOCK = 1; // see psych/nodes/mapping.rb
+        RubyClass stringClass = context.runtime.getString();
+
+        RubyString anchorStr = exportToUTF8(context, anchor, stringClass);
+        RubyString tagStr = exportToUTF8(context, tag, stringClass);
 
         MappingStartEvent event = new MappingStartEvent(
-                anchor.isNil() ? null : anchor.asJavaString(),
-                tag.isNil() ? null : tag.asJavaString(),
+                anchorStr == null ? null : anchorStr.asJavaString(),
+                tagStr == null ? null : tagStr.asJavaString(),
                 implicit.isTrue(),
                 NULL_MARK,
                 NULL_MARK,
                 FLOW_STYLES[style.convertToInteger().getIntValue()]);
+
         emit(context, event);
+
         return this;
     }
 
@@ -263,7 +288,11 @@ public class PsychEmitter extends RubyObject {
     
     @JRubyMethod
     public IRubyObject alias(ThreadContext context, IRubyObject anchor) {
-        AliasEvent event = new AliasEvent(anchor.asJavaString(), NULL_MARK, NULL_MARK);
+        RubyClass stringClass = context.runtime.getString();
+
+        RubyString anchorStr = exportToUTF8(context, anchor, stringClass);
+
+        AliasEvent event = new AliasEvent(anchorStr.asJavaString(), NULL_MARK, NULL_MARK);
         emit(context, event);
         return this;
     }
@@ -278,7 +307,7 @@ public class PsychEmitter extends RubyObject {
     @JRubyMethod
     public IRubyObject canonical(ThreadContext context) {
         // TODO: unclear if this affects a running emitter
-        return context.runtime.newBoolean(options.isCanonical());
+        return RubyBoolean.newBoolean(context, options.isCanonical());
     }
 
     @JRubyMethod(name = "indentation=")
@@ -312,6 +341,9 @@ public class PsychEmitter extends RubyObject {
             if (emitter == null) throw context.runtime.newRuntimeError("uninitialized emitter");
 
             emitter.emit(event);
+
+            // flush writer after each emit
+            writer.flush();
         } catch (IOException ioe) {
             throw context.runtime.newIOErrorFromException(ioe);
         } catch (EmitterException ee) {
@@ -325,10 +357,22 @@ public class PsychEmitter extends RubyObject {
         Encoding encoding = PsychLibrary.YAMLEncoding.values()[(int)_encoding.convertToInteger().getLongValue()].encoding;
         Charset charset = context.runtime.getEncodingService().charsetForEncoding(encoding);
 
-        emitter = new Emitter(new OutputStreamWriter(new IOOutputStream(io, encoding), charset), options);
+        writer = new OutputStreamWriter(new IOOutputStream(io, encoding), charset);
+        emitter = new Emitter(writer, options);
+    }
+
+    private RubyString exportToUTF8(ThreadContext context, IRubyObject tag, RubyClass stringClass) {
+        RubyString tagStr = null;
+        if (!tag.isNil()) {
+            TypeConverter.checkType(context, tag, stringClass);
+            tagStr = (RubyString) tag;
+            tagStr = EncodingUtils.strConvEnc(context, tagStr, tagStr.getEncoding(), UTF8Encoding.INSTANCE);
+        }
+        return tagStr;
     }
 
     Emitter emitter;
+    Writer writer;
     DumperOptions options = new DumperOptions();
     IRubyObject io;
 
diff --git a/ext/java/org/jruby/ext/psych/PsychParser.java b/ext/java/org/jruby/ext/psych/PsychParser.java
index b660a0f..8539110 100644
--- a/ext/java/org/jruby/ext/psych/PsychParser.java
+++ b/ext/java/org/jruby/ext/psych/PsychParser.java
@@ -30,6 +30,9 @@ package org.jruby.ext.psych;
 import java.io.ByteArrayInputStream;
 import java.io.InputStreamReader;
 import java.nio.charset.Charset;
+import java.nio.charset.CharsetDecoder;
+import java.nio.charset.CodingErrorAction;
+import java.nio.charset.MalformedInputException;
 import java.util.Map;
 
 import org.jcodings.Encoding;
@@ -61,6 +64,7 @@ import org.jruby.util.log.LoggerFactory;
 import org.yaml.snakeyaml.DumperOptions;
 import org.yaml.snakeyaml.error.Mark;
 import org.yaml.snakeyaml.error.MarkedYAMLException;
+import org.yaml.snakeyaml.error.YAMLException;
 import org.yaml.snakeyaml.events.AliasEvent;
 import org.yaml.snakeyaml.events.DocumentEndEvent;
 import org.yaml.snakeyaml.events.DocumentStartEvent;
@@ -75,6 +79,8 @@ import org.yaml.snakeyaml.parser.ParserImpl;
 import org.yaml.snakeyaml.reader.ReaderException;
 import org.yaml.snakeyaml.reader.StreamReader;
 import org.yaml.snakeyaml.scanner.ScannerException;
+
+import static org.jruby.runtime.Helpers.arrayOf;
 import static org.jruby.runtime.Helpers.invoke;
 import org.jruby.util.ByteList;
 
@@ -89,8 +95,7 @@ public class PsychParser extends RubyObject {
             }
         }, psych);
 
-        RubyKernel.require(runtime.getNil(),
-                runtime.newString("psych/syntax_error"), Block.NULL_BLOCK);
+        runtime.getLoadService().require("psych/syntax_error");
         psychParser.defineConstant("ANY", runtime.newFixnum(YAML_ANY_ENCODING.ordinal()));
         psychParser.defineConstant("UTF8", runtime.newFixnum(YAML_UTF8_ENCODING.ordinal()));
         psychParser.defineConstant("UTF16LE", runtime.newFixnum(YAML_UTF16LE_ENCODING.ordinal()));
@@ -110,13 +115,15 @@ public class PsychParser extends RubyObject {
         return parse(context, yaml, runtime.getNil());
     }
 
-    private IRubyObject stringOrNilFor(Ruby runtime, String value, boolean tainted) {
-        if (value == null) return runtime.getNil(); // No need to taint nil
+    private IRubyObject stringOrNilFor(ThreadContext context, String value, boolean tainted) {
+        if (value == null) return context.nil;
 
-        return stringFor(runtime, value, tainted);
+        return stringFor(context, value, tainted);
     }
     
-    private RubyString stringFor(Ruby runtime, String value, boolean tainted) {
+    private RubyString stringFor(ThreadContext context, String value, boolean tainted) {
+        Ruby runtime = context.runtime;
+
         Encoding encoding = runtime.getDefaultInternalEncoding();
         if (encoding == null) {
             encoding = UTF8Encoding.INSTANCE;
@@ -136,8 +143,6 @@ public class PsychParser extends RubyObject {
     }
     
     private StreamReader readerFor(ThreadContext context, IRubyObject yaml) {
-        Ruby runtime = context.runtime;
-
         if (yaml instanceof RubyString) {
             ByteList byteList = ((RubyString)yaml).getByteList();
             Encoding enc = byteList.getEncoding();
@@ -175,8 +180,14 @@ public class PsychParser extends RubyObject {
                 // If we can't get it from the IO or it doesn't have a charset, fall back on UTF-8
                 charset = UTF8Encoding.INSTANCE.getCharset();
             }
-            return new StreamReader(new InputStreamReader(new IOInputStream(yaml), charset));
+            CharsetDecoder decoder = charset.newDecoder();
+            decoder.onMalformedInput(CodingErrorAction.REPORT);
+            decoder.onMalformedInput(CodingErrorAction.REPORT);
+
+            return new StreamReader(new InputStreamReader(new IOInputStream(yaml), decoder));
         } else {
+            Ruby runtime = context.runtime;
+
             throw runtime.newTypeError(yaml, runtime.getIO());
         }
     }
@@ -214,7 +225,7 @@ public class PsychParser extends RubyObject {
                     
                     invoke(context, handler, "end_document", notExplicit);
                 } else if (event.is(ID.Alias)) {
-                    IRubyObject alias = stringOrNilFor(runtime, ((AliasEvent)event).getAnchor(), tainted);
+                    IRubyObject alias = stringOrNilFor(context, ((AliasEvent)event).getAnchor(), tainted);
 
                     invoke(context, handler, "alias", alias);
                 } else if (event.is(ID.Scalar)) {
@@ -249,6 +260,16 @@ public class PsychParser extends RubyObject {
             parser = null;
             raiseParserException(context, yaml, re, path);
 
+        } catch (YAMLException ye) {
+            Throwable cause = ye.getCause();
+
+            if (cause instanceof MalformedInputException) {
+                // failure due to improperly encoded input
+                raiseParserException(context, yaml, (MalformedInputException) cause, path);
+            }
+
+            throw ye;
+
         } catch (Throwable t) {
             Helpers.throwException(t);
             return this;
@@ -268,8 +289,8 @@ public class PsychParser extends RubyObject {
         RubyArray tags = RubyArray.newArray(runtime);
         if (tagsMap != null && tagsMap.size() > 0) {
             for (Map.Entry<String, String> tag : tagsMap.entrySet()) {
-                IRubyObject key   = stringFor(runtime, tag.getKey(), tainted);
-                IRubyObject value = stringFor(runtime, tag.getValue(), tainted);
+                IRubyObject key   = stringFor(context, tag.getKey(), tainted);
+                IRubyObject value = stringFor(context, tag.getValue(), tainted);
 
                 tags.append(RubyArray.newArray(runtime, key, value));
             }
@@ -281,8 +302,8 @@ public class PsychParser extends RubyObject {
     
     private void handleMappingStart(ThreadContext context, MappingStartEvent mse, boolean tainted, IRubyObject handler) {
         Ruby runtime = context.runtime;
-        IRubyObject anchor = stringOrNilFor(runtime, mse.getAnchor(), tainted);
-        IRubyObject tag = stringOrNilFor(runtime, mse.getTag(), tainted);
+        IRubyObject anchor = stringOrNilFor(context, mse.getAnchor(), tainted);
+        IRubyObject tag = stringOrNilFor(context, mse.getTag(), tainted);
         IRubyObject implicit = runtime.newBoolean(mse.getImplicit());
         IRubyObject style = runtime.newFixnum(translateFlowStyle(mse.getFlowStyle()));
 
@@ -292,12 +313,12 @@ public class PsychParser extends RubyObject {
     private void handleScalar(ThreadContext context, ScalarEvent se, boolean tainted, IRubyObject handler) {
         Ruby runtime = context.runtime;
 
-        IRubyObject anchor = stringOrNilFor(runtime, se.getAnchor(), tainted);
-        IRubyObject tag = stringOrNilFor(runtime, se.getTag(), tainted);
+        IRubyObject anchor = stringOrNilFor(context, se.getAnchor(), tainted);
+        IRubyObject tag = stringOrNilFor(context, se.getTag(), tainted);
         IRubyObject plain_implicit = runtime.newBoolean(se.getImplicit().canOmitTagInPlainScalar());
         IRubyObject quoted_implicit = runtime.newBoolean(se.getImplicit().canOmitTagInNonPlainScalar());
         IRubyObject style = runtime.newFixnum(translateStyle(se.getScalarStyle()));
-        IRubyObject val = stringFor(runtime, se.getValue(), tainted);
+        IRubyObject val = stringFor(context, se.getValue(), tainted);
 
         invoke(context, handler, "scalar", val, anchor, tag, plain_implicit,
                 quoted_implicit, style);
@@ -305,8 +326,8 @@ public class PsychParser extends RubyObject {
     
     private void handleSequenceStart(ThreadContext context, SequenceStartEvent sse, boolean tainted, IRubyObject handler) {
         Ruby runtime = context.runtime;
-        IRubyObject anchor = stringOrNilFor(runtime, sse.getAnchor(), tainted);
-        IRubyObject tag = stringOrNilFor(runtime, sse.getTag(), tainted);
+        IRubyObject anchor = stringOrNilFor(context, sse.getAnchor(), tainted);
+        IRubyObject tag = stringOrNilFor(context, sse.getTag(), tainted);
         IRubyObject implicit = runtime.newBoolean(sse.getImplicit());
         IRubyObject style = runtime.newFixnum(translateFlowStyle(sse.getFlowStyle()));
 
@@ -360,6 +381,31 @@ public class PsychParser extends RubyObject {
         RubyKernel.raise(context, runtime.getKernel(), new IRubyObject[] { exception }, Block.NULL_BLOCK);
     }
 
+    private static void raiseParserException(ThreadContext context, IRubyObject yaml, MalformedInputException mie, IRubyObject rbPath) {
+        Ruby runtime;
+        Mark mark;
+        RubyClass se;
+        IRubyObject exception;
+
+        runtime = context.runtime;
+        se = (RubyClass)runtime.getModule("Psych").getConstant("SyntaxError");
+
+        mie.getInputLength();
+
+        exception = se.newInstance(context,
+                arrayOf(
+                        rbPath,
+                        runtime.newFixnum(-1),
+                        runtime.newFixnum(-1),
+                        runtime.newFixnum(mie.getInputLength()),
+                        runtime.getNil(),
+                        runtime.getNil()
+                ),
+                Block.NULL_BLOCK);
+
+        RubyKernel.raise(context, runtime.getKernel(), new IRubyObject[] { exception }, Block.NULL_BLOCK);
+    }
+
     private static int translateStyle(DumperOptions.ScalarStyle style) {
         if (style == null) return 0; // any