diff --git a/go.mod b/go.mod index cfac2c3..a92ad6e 100644 --- a/go.mod +++ b/go.mod @@ -1,3 +1,5 @@ module github.com/invopop/yaml -require gopkg.in/yaml.v2 v2.2.2 +go 1.14 + +require gopkg.in/yaml.v3 v3.0.0 diff --git a/go.sum b/go.sum index bd555a3..223abcb 100644 --- a/go.sum +++ b/go.sum @@ -1,3 +1,4 @@ +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= -gopkg.in/yaml.v2 v2.2.2 h1:ZCJp+EgiOT7lHqUV2J862kp8Qj64Jo6az82+3Td9dZw= -gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v3 v3.0.0 h1:hjy8E9ON/egN1tAYqKb61G10WtihqetD4sz2H+8nIeA= +gopkg.in/yaml.v3 v3.0.0/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/yaml.go b/yaml.go index 6a2e53b..66c5c66 100644 --- a/yaml.go +++ b/yaml.go @@ -16,7 +16,7 @@ import ( "reflect" "strconv" - "gopkg.in/yaml.v2" + "gopkg.in/yaml.v3" ) // Marshal the object into JSON then converts JSON to YAML and returns the @@ -41,19 +41,13 @@ type JSONOpt func(*json.Decoder) *json.Decoder // Unmarshal converts YAML to JSON then uses JSON to unmarshal into an object, // optionally configuring the behavior of the JSON unmarshal. func Unmarshal(y []byte, o interface{}, opts ...JSONOpt) error { - return unmarshal(yaml.Unmarshal, y, o, opts) + dec := yaml.NewDecoder(bytes.NewReader(y)) + return unmarshal(dec, o, opts) } -// UnmarshalStrict is like Unmarshal except that any mapping keys that are -// duplicates will result in an error. -// To also be strict about unknown fields, add the DisallowUnknownFields option. -func UnmarshalStrict(y []byte, o interface{}, opts ...JSONOpt) error { - return unmarshal(yaml.UnmarshalStrict, y, o, opts) -} - -func unmarshal(f func(in []byte, out interface{}) (err error), y []byte, o interface{}, opts []JSONOpt) error { +func unmarshal(dec *yaml.Decoder, o interface{}, opts []JSONOpt) error { vo := reflect.ValueOf(o) - j, err := yamlToJSON(y, &vo, f) + j, err := yamlToJSON(dec, &vo) if err != nil { return fmt.Errorf("error converting YAML to JSON: %v", err) } @@ -110,22 +104,15 @@ func JSONToYAML(j []byte) ([]byte, error) { // not use the !!binary tag in your YAML. This will ensure the original base64 // encoded data makes it all the way through to the JSON. // -// For strict decoding of YAML, use YAMLToJSONStrict. func YAMLToJSON(y []byte) ([]byte, error) { //nolint:revive - return yamlToJSON(y, nil, yaml.Unmarshal) -} - -// YAMLToJSONStrict is like YAMLToJSON but enables strict YAML decoding, -// returning an error on any duplicate field names. -func YAMLToJSONStrict(y []byte) ([]byte, error) { //nolint:revive - return yamlToJSON(y, nil, yaml.UnmarshalStrict) + dec := yaml.NewDecoder(bytes.NewReader(y)) + return yamlToJSON(dec, nil) } -func yamlToJSON(y []byte, jsonTarget *reflect.Value, yamlUnmarshal func([]byte, interface{}) error) ([]byte, error) { +func yamlToJSON(dec *yaml.Decoder, jsonTarget *reflect.Value) ([]byte, error) { // Convert the YAML to an object. var yamlObj interface{} - err := yamlUnmarshal(y, &yamlObj) - if err != nil { + if err := dec.Decode(&yamlObj); err != nil { return nil, err } @@ -160,16 +147,13 @@ func convertToJSONableObject(yamlObj interface{}, jsonTarget *reflect.Value) (in } } - // If yamlObj is a number or a boolean, check if jsonTarget is a string - - // if so, coerce. Else return normal. - // If yamlObj is a map or array, find the field that each key is - // unmarshaling to, and when you recurse pass the reflect.Value for that - // field back into this function. + // go-yaml v3 changed from v2 and now will provide map[string]interface{} by + // default and map[interface{}]interface{} when none of the keys strings. + // To get around this, we run a pre-loop to convert the map. + // JSON only supports strings as keys, so we must convert. + switch typedYAMLObj := yamlObj.(type) { case map[interface{}]interface{}: - // JSON does not support arbitrary keys in a map, so we must convert - // these keys to strings. - // // From my reading of go-yaml v2 (specifically the resolve function), // keys can only have the types string, int, int64, float64, binary // (unsupported), or null (unsupported). @@ -188,19 +172,8 @@ func convertToJSONableObject(yamlObj interface{}, jsonTarget *reflect.Value) (in // and 64-bit. Otherwise the key type will simply be int. keyString = strconv.FormatInt(typedKey, 10) case float64: - // Stolen from go-yaml to use the same conversion to string as - // the go-yaml library uses to convert float to string when - // Marshaling. - s := strconv.FormatFloat(typedKey, 'g', -1, 32) - switch s { - case "+Inf": - s = ".inf" - case "-Inf": - s = "-.inf" - case "NaN": - s = ".nan" - } - keyString = s + // Float64 is now supported in keys + keyString = strconv.FormatFloat(typedKey, 'g', -1, 64) case bool: if typedKey { keyString = "true" @@ -211,6 +184,20 @@ func convertToJSONableObject(yamlObj interface{}, jsonTarget *reflect.Value) (in return nil, fmt.Errorf("unsupported map key of type: %s, key: %+#v, value: %+#v", reflect.TypeOf(k), k, v) } + strMap[keyString] = v + } + // replace yamlObj with our new string map + yamlObj = strMap + } + + // If yamlObj is a number or a boolean, check if jsonTarget is a string - + // if so, coerce. Else return normal. + // If yamlObj is a map or array, find the field that each key is + // unmarshaling to, and when you recurse pass the reflect.Value for that + // field back into this function. + switch typedYAMLObj := yamlObj.(type) { + case map[string]interface{}: + for k, v := range typedYAMLObj { // jsonTarget should be a struct or a map. If it's a struct, find // the field it's going to map to and pass its reflect.Value. If @@ -220,7 +207,7 @@ func convertToJSONableObject(yamlObj interface{}, jsonTarget *reflect.Value) (in if jsonTarget != nil { t := *jsonTarget if t.Kind() == reflect.Struct { - keyBytes := []byte(keyString) + keyBytes := []byte(k) // Find the field that the JSON library would use. var f *field fields := cachedTypeFields(t.Type()) @@ -239,7 +226,7 @@ func convertToJSONableObject(yamlObj interface{}, jsonTarget *reflect.Value) (in // Find the reflect.Value of the most preferential // struct field. jtf := t.Field(f.index[0]) - strMap[keyString], err = convertToJSONableObject(v, &jtf) + typedYAMLObj[k], err = convertToJSONableObject(v, &jtf) if err != nil { return nil, err } @@ -249,19 +236,19 @@ func convertToJSONableObject(yamlObj interface{}, jsonTarget *reflect.Value) (in // Create a zero value of the map's element type to use as // the JSON target. jtv := reflect.Zero(t.Type().Elem()) - strMap[keyString], err = convertToJSONableObject(v, &jtv) + typedYAMLObj[k], err = convertToJSONableObject(v, &jtv) if err != nil { return nil, err } continue } } - strMap[keyString], err = convertToJSONableObject(v, nil) + typedYAMLObj[k], err = convertToJSONableObject(v, nil) if err != nil { return nil, err } } - return strMap, nil + return typedYAMLObj, nil case []interface{}: // We need to recurse into arrays in case there are any // map[interface{}]interface{}'s inside and to convert any @@ -303,7 +290,7 @@ func convertToJSONableObject(yamlObj interface{}, jsonTarget *reflect.Value) (in case int64: s = strconv.FormatInt(typedVal, 10) case float64: - s = strconv.FormatFloat(typedVal, 'g', -1, 32) + s = strconv.FormatFloat(typedVal, 'g', -1, 64) case uint64: s = strconv.FormatUint(typedVal, 10) case bool: diff --git a/yaml_test.go b/yaml_test.go index 9485ff1..749aca4 100644 --- a/yaml_test.go +++ b/yaml_test.go @@ -13,15 +13,15 @@ import ( type MarshalTest struct { A string B int64 - // Would like to test float64, but it's not supported in go-yaml. - // (See https://github.com/go-yaml/yaml/issues/83.) C float32 + D float64 } func TestMarshal(t *testing.T) { f32String := strconv.FormatFloat(math.MaxFloat32, 'g', -1, 32) - s := MarshalTest{"a", math.MaxInt64, math.MaxFloat32} - e := []byte(fmt.Sprintf("A: a\nB: %d\nC: %s\n", int64(math.MaxInt64), f32String)) + f64String := strconv.FormatFloat(math.MaxFloat64, 'g', -1, 64) + s := MarshalTest{"a", math.MaxInt64, math.MaxFloat32, math.MaxFloat64} + e := []byte(fmt.Sprintf("A: a\nB: %d\nC: %s\nD: %s\n", int64(math.MaxInt64), f32String, f64String)) y, err := Marshal(s) if err != nil { @@ -35,8 +35,8 @@ func TestMarshal(t *testing.T) { } type UnmarshalString struct { - A string - True string + A string + B string } type UnmarshalStringMap struct { @@ -66,14 +66,19 @@ func TestUnmarshal(t *testing.T) { e1 := UnmarshalString{A: "1"} unmarshalEqual(t, y, &s1, &e1) + y = []byte(`a: "1"`) + s1 = UnmarshalString{} + e1 = UnmarshalString{A: "1"} + unmarshalEqual(t, y, &s1, &e1) + y = []byte("a: true") s1 = UnmarshalString{} e1 = UnmarshalString{A: "true"} unmarshalEqual(t, y, &s1, &e1) - y = []byte("true: 1") + y = []byte("a: 1") s1 = UnmarshalString{} - e1 = UnmarshalString{True: "1"} + e1 = UnmarshalString{A: "1"} unmarshalEqual(t, y, &s1, &e1) y = []byte("a:\n a: 1") @@ -118,6 +123,11 @@ func TestUnmarshalNonStrict(t *testing.T) { yaml: []byte("a: 1"), want: UnmarshalString{A: "1"}, }, + { + // Order does not matter. + yaml: []byte("b: 1\na: 2"), + want: UnmarshalString{A: "2", B: "1"}, + }, { // Unknown field get ignored. yaml: []byte("a: 1\nunknownField: 2"), @@ -129,24 +139,9 @@ func TestUnmarshalNonStrict(t *testing.T) { want: UnmarshalString{A: "1"}, }, { - // Last declaration of `a` wins. - yaml: []byte("a: 1\na: 2"), - want: UnmarshalString{A: "2"}, - }, - { - // Even ignore first declaration of `a` with wrong type. - yaml: []byte("a: [1,2,3]\na: value-of-a"), - want: UnmarshalString{A: "value-of-a"}, - }, - { - // Last value of `a` and first and only mention of `true` are parsed. - yaml: []byte("true: string-value-of-yes\na: 1\na: [1,2,3]\na: value-of-a"), - want: UnmarshalString{A: "value-of-a", True: "string-value-of-yes"}, - }, - { - // In YAML, `YES` is a Boolean true. - yaml: []byte("true: YES"), - want: UnmarshalString{True: "true"}, + // In YAML, `YES` is no longer Boolean true. + yaml: []byte("a: YES"), + want: UnmarshalString{A: "YES"}, }, } { s := UnmarshalString{} @@ -177,72 +172,41 @@ func unmarshalEqual(t *testing.T, y []byte, s, e interface{}, opts ...JSONOpt) { } } -// TestUnmarshalStrict tests that we return an error on ambiguous YAML. -func TestUnmarshalStrict(t *testing.T) { +// TestUnmarshalErrors tests that we return an error on ambiguous YAML. +func TestUnmarshalErrors(t *testing.T) { for _, tc := range []struct { yaml []byte - want UnmarshalString wantErr string }{ - { - yaml: []byte("a: 1"), - want: UnmarshalString{A: "1"}, - }, - { - // Order does not matter. - yaml: []byte("true: 1\na: 2"), - want: UnmarshalString{A: "2", True: "1"}, - }, - { - // By default, unknown field is ignored. - yaml: []byte("a: 1\nunknownField: 2"), - want: UnmarshalString{A: "1"}, - }, { // Declaring `a` twice produces an error. yaml: []byte("a: 1\na: 2"), - wantErr: `key "a" already set in map`, + wantErr: `key "a" already defined`, }, { // Not ignoring first declaration of A with wrong type. yaml: []byte("a: [1,2,3]\na: value-of-a"), - wantErr: `key "a" already set in map`, + wantErr: `key "a" already defined`, }, { // Declaring field `true` twice. yaml: []byte("true: string-value-of-yes\ntrue: 1"), - wantErr: `key true already set in map`, - }, - { - // In YAML, `YES` is a Boolean true. - yaml: []byte("true: YES"), - want: UnmarshalString{True: "true"}, + wantErr: `key "true" already defined`, }, } { s := UnmarshalString{} - err := UnmarshalStrict(tc.yaml, &s) + err := Unmarshal(tc.yaml, &s) if tc.wantErr != "" && err == nil { - t.Errorf("UnmarshalStrict(%#q, &s) = nil; want error", string(tc.yaml)) + t.Errorf("Unmarshal(%#q, &s) = nil; want error", string(tc.yaml)) continue } if tc.wantErr == "" && err != nil { - t.Errorf("UnmarshalStrict(%#q, &s) = %v; want no error", string(tc.yaml), err) + t.Errorf("Unmarshal(%#q, &s) = %v; want no error", string(tc.yaml), err) continue } // We only expect errors during unmarshalling YAML. - if want := "yaml: unmarshal errors"; tc.wantErr != "" && !strings.Contains(err.Error(), want) { - t.Errorf("UnmarshalStrict(%#q, &s) = %v; want err contains %#q", string(tc.yaml), err, want) - } if tc.wantErr != "" && !strings.Contains(err.Error(), tc.wantErr) { - t.Errorf("UnmarshalStrict(%#q, &s) = %v; want err contains %#q", string(tc.yaml), err, tc.wantErr) - } - - // Even if there was an error, we continue the test: We expect that all - // errors occur during YAML unmarshalling. Such errors leaves `s` unmodified - // and the following check will compare default values of `UnmarshalString`. - - if !reflect.DeepEqual(s, tc.want) { - t.Errorf("UnmarshalStrict(%#q, &s) = %+#v; want %+#v", string(tc.yaml), s, tc.want) + t.Errorf("Unmarshal(%#q, &s) = %v; want err contains %#q", string(tc.yaml), err, tc.wantErr) } } } @@ -292,7 +256,18 @@ func TestYAMLToJSON(t *testing.T) { "t: null\n", `{"t":null}`, nil, - }, { + }, + { + "true: yes\n", + `{"true":"yes"}`, + strPtr("\"true\": \"yes\"\n"), + }, + { + "false: yes\n", + `{"false":"yes"}`, + strPtr("\"false\": \"yes\"\n"), + }, + { "1: a\n", `{"1":"a"}`, strPtr("\"1\": a\n"), @@ -415,15 +390,9 @@ func strPtr(s string) *string { return &s } -func TestYAMLToJSONStrict(t *testing.T) { - const data = ` -foo: bar -foo: baz -` - if _, err := YAMLToJSON([]byte(data)); err != nil { - t.Error("expected YAMLtoJSON to pass on duplicate field names") - } - if _, err := YAMLToJSONStrict([]byte(data)); err == nil { - t.Error("expected YAMLtoJSONStrict to fail on duplicate field names") +func TestYAMLToJSONDuplicateFields(t *testing.T) { + data := []byte("foo: bar\nfoo: baz\n") + if _, err := YAMLToJSON(data); err == nil { + t.Error("expected YAMLtoJSON to fail on duplicate field names") } }