From d083650302691c411829ac10007e8c89cc953420 Mon Sep 17 00:00:00 2001 From: upstream Date: Fri, 4 Sep 2026 07:41:12 +0200 Subject: [PATCH] Upstream gocarina/gocsv state of the touched files before ed8a0671ef --- decode_test.go | 1361 ++++++++++++++++++++++++++++++++++++++++++++++++ types.go | 493 ++++++++++++++++++ types_test.go | 141 +++++ 3 files changed, 1995 insertions(+) create mode 100644 decode_test.go create mode 100644 types.go create mode 100644 types_test.go diff --git a/decode_test.go b/decode_test.go new file mode 100644 index 0000000..792cc3a --- /dev/null +++ b/decode_test.go @@ -0,0 +1,1361 @@ +package gocsv + +import ( + "bytes" + "encoding/csv" + "errors" + "io" + "reflect" + "strconv" + "strings" + "testing" + "time" +) + +func Test_readTo(t *testing.T) { + blah := 0 + sptr := "*string" + sptr2 := "" + b := bytes.NewBufferString(`foo,BAR,Baz,Blah,SPtr,Omit +f,1,baz,,*string,*string +e,3,b,,,`) + d := newSimpleDecoderFromReader(b) + + var samples []Sample + if err := readTo(d, &samples); err != nil { + t.Fatal(err) + } + if len(samples) != 2 { + t.Fatalf("expected 2 sample instances, got %d", len(samples)) + } + + expected := Sample{Foo: "f", Bar: 1, Baz: "baz", Blah: &blah, SPtr: &sptr, Omit: &sptr} + if !reflect.DeepEqual(expected, samples[0]) { + t.Fatalf("expected first sample %v, got %v", expected, samples[0]) + } + + expected = Sample{Foo: "e", Bar: 3, Baz: "b", Blah: &blah, SPtr: &sptr2} + if !reflect.DeepEqual(expected, samples[1]) { + t.Fatalf("expected second sample %v, got %v", expected, samples[1]) + } + + b = bytes.NewBufferString(`foo,BAR,Baz +f,1,baz +e,BAD_INPUT,b`) + d = newSimpleDecoderFromReader(b) + samples = []Sample{} + err := readTo(d, &samples) + if err == nil { + t.Fatalf("Expected error from bad input, got: %+v", samples) + } + switch actualErr := err.(type) { + case *csv.ParseError: + if actualErr.Line != 3 { + t.Fatalf("Expected csv.ParseError on line 3, got: %d", actualErr.Line) + } + if actualErr.Column != 2 { + t.Fatalf("Expected csv.ParseError in column 2, got: %d", actualErr.Column) + } + default: + t.Fatalf("incorrect error type: %T", err) + } + +} + +func Test_readToNormalized(t *testing.T) { + SetHeaderNormalizer(func(s string) string { + return strings.ToLower(s) + }) + defer SetHeaderNormalizer(DefaultNameNormalizer()) + + blah := 0 + sptr := "*string" + sptr2 := "" + b := bytes.NewBufferString(`FOO,BAR,BAZ,BLAH,SPTR,OMIT +f,1,baz,,*string,*string +e,3,b,,,`) + d := newSimpleDecoderFromReader(b) + + var samples []Sample + if err := readTo(d, &samples); err != nil { + t.Fatal(err) + } + if len(samples) != 2 { + t.Fatalf("expected 2 sample instances, got %d", len(samples)) + } + + expected := Sample{Foo: "f", Bar: 1, Baz: "baz", Blah: &blah, SPtr: &sptr, Omit: &sptr} + if !reflect.DeepEqual(expected, samples[0]) { + t.Fatalf("expected first sample %v, got %v", expected, samples[0]) + } + + expected = Sample{Foo: "e", Bar: 3, Baz: "b", Blah: &blah, SPtr: &sptr2} + if !reflect.DeepEqual(expected, samples[1]) { + t.Fatalf("expected second sample %v, got %v", expected, samples[1]) + } + + b = bytes.NewBufferString(`foo,BAR,Baz +f,1,baz +e,BAD_INPUT,b`) + d = newSimpleDecoderFromReader(b) + samples = []Sample{} + err := readTo(d, &samples) + if err == nil { + t.Fatalf("Expected error from bad input, got: %+v", samples) + } + switch actualErr := err.(type) { + case *csv.ParseError: + if actualErr.Line != 3 { + t.Fatalf("Expected csv.ParseError on line 3, got: %d", actualErr.Line) + } + if actualErr.Column != 2 { + t.Fatalf("Expected csv.ParseError in column 2, got: %d", actualErr.Column) + } + default: + t.Fatalf("incorrect error type: %T", err) + } + +} + +func Test_readTo_Time(t *testing.T) { + b := bytes.NewBufferString(`Foo +1970-01-01T03:01:00+03:00`) + d := newSimpleDecoderFromReader(b) + + var samples []DateTime + if err := readTo(d, &samples); err != nil { + t.Fatal(err) + } + + rt, _ := time.Parse(time.RFC3339, "1970-01-01T03:01:00+03:00") + + expected := DateTime{Foo: rt} + + if !reflect.DeepEqual(expected, samples[0]) { + t.Fatalf("expected first sample %v, got %v", expected, samples[0]) + } +} + +func Test_readTo_complex_embed(t *testing.T) { + b := bytes.NewBufferString(`first,foo,BAR,Baz,last,abc +aa,bb,11,cc,dd,ee +ff,gg,22,hh,ii,jj`) + d := newSimpleDecoderFromReader(b) + + var samples []SkipFieldSample + if err := readTo(d, &samples); err != nil { + t.Fatal(err) + } + if len(samples) != 2 { + t.Fatalf("expected 2 sample instances, got %d", len(samples)) + } + expected := SkipFieldSample{ + EmbedSample: EmbedSample{ + Qux: "aa", + Sample: Sample{ + Foo: "bb", + Bar: 11, + Baz: "cc", + }, + Quux: "dd", + }, + Corge: "ee", + } + if expected != samples[0] { + t.Fatalf("expected first sample %v, got %v", expected, samples[0]) + } + expected = SkipFieldSample{ + EmbedSample: EmbedSample{ + Qux: "ff", + Sample: Sample{ + Foo: "gg", + Bar: 22, + Baz: "hh", + }, + Quux: "ii", + }, + Corge: "jj", + } + if expected != samples[1] { + t.Fatalf("expected first sample %v, got %v", expected, samples[1]) + } +} + +func Test_readTo_embed_ptr(t *testing.T) { + b := bytes.NewBufferString(`first,foo,BAR,Baz,last,abc +aa,bb,11,cc,dd,ee +ff,gg,22,hh,ii,jj`) + d := newSimpleDecoderFromReader(b) + var rows []EmbedPtrSample + if err := readTo(d, &rows); err != nil { + t.Fatalf(err.Error()) + } + expected := EmbedPtrSample{ + Qux: "ff", + Sample: &Sample{ + Foo: "gg", + Bar: 22, + Baz: "hh", + }, + Quux: "ii", + } + if !reflect.DeepEqual(expected, rows[1]) { + t.Fatalf("expected first sample %v, got %+v", expected, rows[1]) + } +} + +func Test_readTo_slice(t *testing.T) { + b := bytes.NewBufferString(`Slice +[] +[1, 2, 3]`) + reader := csv.NewReader(b) + reader.Comma = '\t' + d := csvDecoder{reader} + samples := []SliceSample{} + if err := readTo(d, &samples); err != nil { + t.Fatal(err) + } + expected := SliceSample{Slice: []int{}} + if !reflect.DeepEqual(expected, samples[0]) { + t.Fatalf("expected first sample %v, got %v", expected, samples[0].Slice) + } + expected = SliceSample{Slice: []int{1, 2, 3}} + if !reflect.DeepEqual(expected, samples[1]) { + t.Fatalf("expected second sample %v, got %v", expected, samples[1].Slice) + } +} + +func Test_readTo_slice_structs(t *testing.T) { + b := bytes.NewBufferString(`s[0].string,slice[0].f,slice[1].s,s[1].float,a[0].s,array[0].float,a[1].s,array[1].float,ints[0],ints[1],ints[2] +s1,1.1,s2,2.2,s3,3.3,s4,4.4,1,2,3`) + d := newSimpleDecoderFromReader(b) + + var samples []SliceStructSample + err := readTo(d, &samples) + if err != nil { + t.Fatalf("failed to unmarshal: %v", err) + } + + expected := SliceStructSample{ + Slice: []SliceStruct{ + {String: "s1", Float: 1.1}, + {String: "s2", Float: 2.2}, + }, + SimpleSlice: []int{1, 2, 3}, + Array: [2]SliceStruct{ + {String: "s3", Float: 3.3}, + {String: "s4", Float: 4.4}, + }, + } + + if !reflect.DeepEqual(expected, samples[0]) { + t.Fatalf("expected \n sample: %v\n got: %v", expected, samples[0]) + } +} + +func Test_readTo_embed_marshal(t *testing.T) { + b := bytes.NewBufferString(`foo +bar`) + d := newSimpleDecoderFromReader(b) + var rows []EmbedMarshal + if err := readTo(d, &rows); err != nil { + t.Fatalf(err.Error()) + } + expected := EmbedMarshal{ + Foo: &MarshalSample{Dummy: "bar"}, + } + if !reflect.DeepEqual(expected, rows[0]) { + t.Fatalf("expected first sample %v, got %+v", expected, rows[1]) + } +} + +func Test_readTo_embed_unmarshal_csv_with_clashing_field(t *testing.T) { + b := bytes.NewBufferString(`Symbol,Timestamp +test,1656460798.693201614`) + d := newSimpleDecoderFromReader(b) + var rows []EmbedUnmarshalCSVWithClashingField + if err := readTo(d, &rows); err != nil { + t.Fatalf(err.Error()) + } + expected := EmbedUnmarshalCSVWithClashingField{ + Symbol: "test", + Timestamp: &UnmarshalCSVSample{Timestamp: 1656460798, Nanos: 693201614}, + } + if !reflect.DeepEqual(expected, rows[0]) { + t.Fatalf("expected first sample %v, got %+v", expected, rows[0]) + } +} + +func Test_readEach(t *testing.T) { + b := bytes.NewBufferString(`first,foo,BAR,Baz,last,abc +aa,bb,11,cc,dd,ee +ff,gg,22,hh,ii,jj`) + d := newSimpleDecoderFromReader(b) + + c := make(chan SkipFieldSample) + e := make(chan error) + var samples []SkipFieldSample + go func() { + if err := readEach(d, nil, c); err != nil { + e <- err + } + }() +L: + for { + select { + case err := <-e: + t.Fatal(err) + case v, ok := <-c: + if !ok { + break L + } + samples = append(samples, v) + + } + } + if len(samples) != 2 { + t.Fatalf("expected 2 sample instances, got %d", len(samples)) + } + expected := SkipFieldSample{ + EmbedSample: EmbedSample{ + Qux: "aa", + Sample: Sample{ + Foo: "bb", + Bar: 11, + Baz: "cc", + }, + Quux: "dd", + }, + Corge: "ee", + } + if expected != samples[0] { + t.Fatalf("expected first sample %v, got %v", expected, samples[0]) + } + expected = SkipFieldSample{ + EmbedSample: EmbedSample{ + Qux: "ff", + Sample: Sample{ + Foo: "gg", + Bar: 22, + Baz: "hh", + }, + Quux: "ii", + }, + Corge: "jj", + } + if expected != samples[1] { + t.Fatalf("expected first sample %v, got %v", expected, samples[1]) + } +} + +func Test_readEach_withErrorHandler(t *testing.T) { + b := bytes.NewBufferString(`first,foo,BAR,Baz,last,abc +aa,bb,11,cc,dd,ee +ff,gg,22,hh,ii,jj +kk,ll,ab,mm,nn,oo +`) + d := newSimpleDecoderFromReader(b) + + var errHandler ErrorHandler + errHandler = func(parseError *csv.ParseError) bool { + if parseError.Line != 4 { + t.Fatalf("expected parse error on line 4, got %d", parseError.Line) + } + + if parseError.Column != 3 { + t.Fatalf("expected parse error on column 3, got %d", parseError.Column) + } + return false + } + + c := make(chan SkipFieldSample) + e := make(chan error) + var samples []SkipFieldSample + go func() { + if err := readEach(d, errHandler, c); err != nil { + e <- err + } + }() +L: + for { + select { + case err := <-e: + t.Fatal(err) + case v, ok := <-c: + if !ok { + break L + } + samples = append(samples, v) + + } + } + if len(samples) != 2 { + t.Fatalf("expected 2 sample instances, got %d", len(samples)) + } + expected := SkipFieldSample{ + EmbedSample: EmbedSample{ + Qux: "aa", + Sample: Sample{ + Foo: "bb", + Bar: 11, + Baz: "cc", + }, + Quux: "dd", + }, + Corge: "ee", + } + if expected != samples[0] { + t.Fatalf("expected first sample %v, got %v", expected, samples[0]) + } + expected = SkipFieldSample{ + EmbedSample: EmbedSample{ + Qux: "ff", + Sample: Sample{ + Foo: "gg", + Bar: 22, + Baz: "hh", + }, + Quux: "ii", + }, + Corge: "jj", + } + if expected != samples[1] { + t.Fatalf("expected first sample %v, got %v", expected, samples[1]) + } +} + +func Test_readEachWithoutHeaders(t *testing.T) { + blah := 0 + sptr := "" + b := bytes.NewBufferString(`f,1,baz,1.66,,, +e,3,b,,,,`) + d := newSimpleDecoderFromReader(b) + + c := make(chan Sample) + e := make(chan error) + var samples []Sample + go func() { + if err := readEachWithoutHeaders(d, c); err != nil { + e <- err + } + }() +L: + for { + select { + case err := <-e: + t.Fatal(err) + case v, ok := <-c: + if !ok { + break L + } + samples = append(samples, v) + + } + } + if len(samples) != 2 { + t.Fatalf("expected 2 sample instances, got %d", len(samples)) + } + + expected := Sample{Foo: "f", Bar: 1, Baz: "baz", Frop: 1.66, Blah: &blah, SPtr: &sptr} + if !reflect.DeepEqual(expected, samples[0]) { + t.Fatalf("expected first sample %v, got %v", expected, samples[0]) + } + + expected = Sample{Foo: "e", Bar: 3, Baz: "b", Frop: 0, Blah: &blah, SPtr: &sptr} + if !reflect.DeepEqual(expected, samples[1]) { + t.Fatalf("expected second sample %v, got %v", expected, samples[1]) + } +} + +// Test_readEach_TypeUnmarshalCSVWithFields test that readEach works with type implementing TypeUnmarshalCSVWithFields for unmarshalling +func Test_readEach_TypeUnmarshalCSVWithFields(t *testing.T) { + b := bytes.NewBufferString(`foo,bar,baz,frop +bb,1,cc,3.14 +gg,2,hh,4`) + d := newSimpleDecoderFromReader(b) + + c := make(chan UnmarshalCSVWithFieldsSample) + e := make(chan error) + var samples []UnmarshalCSVWithFieldsSample + go func() { + if err := readEach(d, nil, c); err != nil { + e <- err + } + }() +L: + for { + select { + case err := <-e: + t.Fatal(err) + case v, ok := <-c: + if !ok { + break L + } + samples = append(samples, v) + + } + } + if len(samples) != 2 { + t.Fatalf("expected 2 sample instances, got %d", len(samples)) + } + expected := UnmarshalCSVWithFieldsSample{ + Foo: "bb", + Bar: 1, + Baz: "cc", + Frop: 314, + } + if expected != samples[0] { + t.Fatalf("expected first sample %v, got %v", expected, samples[0]) + } + expected = UnmarshalCSVWithFieldsSample{ + Foo: "gg", + Bar: 2, + Baz: "hh", + Frop: 400, + } + if expected != samples[1] { + t.Fatalf("expected first sample %v, got %v", expected, samples[1]) + } +} + +func Test_maybeMissingStructFields(t *testing.T) { + structTags := []fieldInfo{ + {keys: []string{"foo"}}, + {keys: []string{"bar"}}, + {keys: []string{"baz"}}, + } + badHeaders := []string{"hi", "mom", "bacon"} + goodHeaders := []string{"foo", "bar", "baz"} + + // no tags to match, expect no error + if err := maybeMissingStructFields([]fieldInfo{}, goodHeaders); err != nil { + t.Fatal(err) + } + + // bad headers, expect an error + if err := maybeMissingStructFields(structTags, badHeaders); err == nil { + t.Fatal("expected an error, but no error found") + } else if !errors.Is(err, ErrUnmatchedStructTags) { + t.Fatal("expected ErrUnmatchedStructTags, but got a different error") + } + + // good headers, expect no error + if err := maybeMissingStructFields(structTags, goodHeaders); err != nil { + t.Fatal(err) + } + + // extra headers, but all structtags match; expect no error + moarHeaders := append(goodHeaders, "qux", "quux", "corge", "grault") + if err := maybeMissingStructFields(structTags, moarHeaders); err != nil { + t.Fatal(err) + } + + // not all structTags match, but there's plenty o' headers; expect + // error + mismatchedHeaders := []string{"foo", "qux", "quux", "corgi"} + if err := maybeMissingStructFields(structTags, mismatchedHeaders); err == nil { + t.Fatal("expected an error, but no error found") + } else if !errors.Is(err, ErrUnmatchedStructTags) { + t.Fatal("expected ErrUnmatchedStructTags, but got a different error") + } +} + +func Test_maybeDoubleHeaderNames(t *testing.T) { + b := bytes.NewBufferString(`foo,BAR,foo +f,1,baz +e,3,b`) + d := newSimpleDecoderFromReader(b) + var samples []Sample + + // *** check maybeDoubleHeaderNames + if err := maybeDoubleHeaderNames([]string{"foo", "BAR", "foo"}); err == nil { + t.Fatal("maybeDoubleHeaderNames did not raise an error when a should have.") + } else if !errors.Is(err, ErrDoubleHeaderNames) { + t.Fatal("maybeDoubleHeaderNames did not raise an error of type ErrDoubleHeaderNames.") + } + + // *** check readTo + if err := readTo(d, &samples); err != nil { + t.Fatal(err) + } + // Double header allowed, value should be of third row + if samples[0].Foo != "baz" { + t.Fatal("Double header allowed, value should be of third row but is not. Function called is readTo.") + } + + b = bytes.NewBufferString(`foo,BAR,foo +f,1,baz +e,3,b`) + d = newSimpleDecoderFromReader(b) + ShouldAlignDuplicateHeadersWithStructFieldOrder = true + if err := readTo(d, &samples); err != nil { + t.Fatal(err) + } + // Double header allowed, value should be of first row + if samples[0].Foo != "f" { + t.Fatal("Double header allowed, value should be of first row but is not. Function called is readTo.") + } + + ShouldAlignDuplicateHeadersWithStructFieldOrder = false + // Double header not allowed, should fail + FailIfDoubleHeaderNames = true + if err := readTo(d, &samples); err == nil { + t.Fatal("Double header not allowed but no error raised. Function called is readTo.") + } + + // *** check readEach + FailIfDoubleHeaderNames = false + b = bytes.NewBufferString(`foo,BAR,foo +f,1,baz +e,3,b`) + d = newSimpleDecoderFromReader(b) + samples = samples[:0] + c := make(chan Sample) + e := make(chan error) + go func() { + if err := readEach(d, nil, c); err != nil { + e <- err + } + }() +L1: + for { + select { + case err := <-e: + t.Fatal(err) + case v, ok := <-c: + if !ok { + break L1 + } + samples = append(samples, v) + + } + } + // Double header allowed, value should be of third row + if samples[0].Foo != "baz" { + t.Fatal("Double header allowed, value should be of third row but is not. Function called is readEach.") + } + // Double header not allowed, should fail + FailIfDoubleHeaderNames = true + b = bytes.NewBufferString(`foo,BAR,foo +f,1,baz +e,3,b`) + d = newSimpleDecoderFromReader(b) + c = make(chan Sample) + e = make(chan error) + go func() { + if err := readEach(d, nil, c); err == nil { + e <- err + } + }() +L2: + for { + select { + case <-e: + t.Fatal("Double header not allowed but no error raised. Function called is readEach.") + case v, ok := <-c: + if !ok { + break L2 + } + samples = append(samples, v) + + } + } +} + +func TestUnmarshalToCallback(t *testing.T) { + b := bytes.NewBufferString(`first,foo,BAR,Baz,last,abc +aa,bb,11,cc,dd,ee +ff,gg,22,hh,ii,jj`) + var samples []SkipFieldSample + if err := UnmarshalBytesToCallback(b.Bytes(), func(s SkipFieldSample) { + samples = append(samples, s) + }); err != nil { + t.Fatal(err) + } + if len(samples) != 2 { + t.Fatalf("expected 2 sample instances, got %d", len(samples)) + } + expected := SkipFieldSample{ + EmbedSample: EmbedSample{ + Qux: "aa", + Sample: Sample{ + Foo: "bb", + Bar: 11, + Baz: "cc", + }, + Quux: "dd", + }, + Corge: "ee", + } + if expected != samples[0] { + t.Fatalf("expected first sample %v, got %v", expected, samples[0]) + } + expected = SkipFieldSample{ + EmbedSample: EmbedSample{ + Qux: "ff", + Sample: Sample{ + Foo: "gg", + Bar: 22, + Baz: "hh", + }, + Quux: "ii", + }, + Corge: "jj", + } + if expected != samples[1] { + t.Fatalf("expected first sample %v, got %v", expected, samples[1]) + } +} + +// TestRenamedTypes tests for unmarshaling functions on redefined basic types. +func TestRenamedTypesUnmarshal(t *testing.T) { + // Set different csv field separator to enable comma in floats + SetCSVReader(func(in io.Reader) CSVReader { + csvin := csv.NewReader(in) + csvin.Comma = ';' + return csvin + }) + + b := bytes.NewBufferString(`foo;bar +1,4;1.5 +2,3;2.4`) + d := newSimpleDecoderFromReader(b) + var samples []RenamedSample + + // Switch back to default for tests executed after this + defer SetCSVReader(DefaultCSVReader) + + if err := readTo(d, &samples); err != nil { + t.Fatal(err) + } + if samples[0].RenamedFloatUnmarshaler != 1.4 { + t.Fatalf("Parsed float value wrong for renamed float64 type. Expected 1.4, got %v.", samples[0].RenamedFloatUnmarshaler) + } + if samples[0].RenamedFloatDefault != 1.5 { + t.Fatalf("Parsed float value wrong for renamed float64 type without an explicit unmarshaler function. Expected 1.5, got %v.", samples[0].RenamedFloatDefault) + } + + // Test that errors raised by UnmarshalCSV are correctly reported + b = bytes.NewBufferString(`foo;bar +4.2;2.4`) + d = newSimpleDecoderFromReader(b) + samples = samples[:0] + if perr, _ := readTo(d, &samples).(*csv.ParseError); perr == nil { + t.Fatalf("Expected ParseError, got nil.") + } else if _, ok := perr.Err.(UnmarshalError); !ok { + t.Fatalf("Expected UnmarshalError, got %v", perr.Err) + } +} + +func (rf *RenamedFloat64Unmarshaler) UnmarshalCSV(csv string) (err error) { + // Purely for testing purposes: Raise error on specific string + if csv == "4.2" { + return UnmarshalError{"Test error: Invalid float 4.2"} + } + + // Convert , to . before parsing to create valid float strings + converted := strings.Replace(csv, ",", ".", -1) + var f float64 + if f, err = strconv.ParseFloat(converted, 64); err != nil { + return err + } + *rf = RenamedFloat64Unmarshaler(f) + return nil +} + +// TestUnmarshalCSVWithFields test that the TestUnmarshalCSVWithFields interface to unmarshal all the fields works +func TestUnmarshalCSVWithFields(t *testing.T) { + b := []byte(`foo,bar,baz,frop +bar,1,zip,3.14 +baz,2,zap,4.00`) + var samples []UnmarshalCSVWithFieldsSample + err := UnmarshalBytes(b, &samples) + if err != nil { + t.Fatalf("UnmarshalCSVWithFields() -> UnmarshalBytes() %v", err) + } + + tests := []struct { + name string + index int + wantFoo string + wantBar int + wantBaz string + wantFrop float64 + }{ + { + name: "should validate index 0", + index: 0, + wantFoo: "bar", + wantBar: 1, + wantBaz: "zip", + wantFrop: 314, + }, + { + name: "should validate index 1", + index: 1, + wantFoo: "baz", + wantBar: 2, + wantBaz: "zap", + wantFrop: 400, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if samples[tt.index].Foo != tt.wantFoo { + t.Fatalf("UnmarshalCSVWithFields() Index %d Foo expected %v got %v", tt.index, tt.wantFoo, samples[tt.index].Foo) + } + + if samples[tt.index].Bar != tt.wantBar { + t.Fatalf("UnmarshalCSVWithFields() Index %d Bar expected %v got %v", tt.index, tt.wantBar, samples[tt.index].Bar) + } + + if samples[tt.index].Baz != tt.wantBaz { + t.Fatalf("UnmarshalCSVWithFields() Index %d Baz expected %v got %v", tt.index, tt.wantBaz, samples[tt.index].Baz) + } + + if samples[tt.index].Frop != tt.wantFrop { + t.Fatalf("UnmarshalCSVWithFields() Index %d Frop expected %v got %v", tt.index, tt.wantFrop, samples[tt.index].Frop) + } + + }) + } +} + +func (u *UnmarshalCSVWithFieldsSample) UnmarshalCSVWithFields(key, value string) error { + switch key { + case "foo": + u.Foo = value + case "bar": + i, err := strconv.Atoi(value) + if err != nil { + return err + } + u.Bar = i + case "baz": + u.Baz = value + case "frop": + f, err := strconv.ParseFloat(value, 64) + if err != nil { + return err + } + u.Frop = f * 100 + } + return nil +} + +type UnmarshalError struct { + msg string +} + +func (e UnmarshalError) Error() string { + return e.msg +} + +// TestUnmarshalCSVWithFieldsNoTags test that the TypeUnmarshalCSVWithFields interface works for types without matching csv tags (or no csv tags at all) +func TestUnmarshalCSVWithFieldsNoTags(t *testing.T) { + b := []byte(`foo,bar,baz,frop +bar,1,zip,3.14 +baz,2,zap,4.00`) + var samples []NoTagsSample + err := UnmarshalBytes(b, &samples) + if err != nil { + t.Fatalf("UnmarshalCSVWithFields() -> UnmarshalBytes() %v", err) + } + + if len(samples) != 2 { + t.Fatalf("expected 2 sample instances, got %d", len(samples)) + } + + if len(samples[0].Fields) != 4 { + t.Fatalf("expected 4 sample fields in map, got %d", len(samples[0].Fields)) + } + + checkFieldMap := func(expected map[string]string, actual map[string]string) { + for expectedKey, expectedValue := range expected { + if sampleValue, ok := actual[expectedKey]; ok { + if sampleValue != expectedValue { + t.Fatalf("expected %s key in map to have %s value, got %s", expectedKey, expectedValue, sampleValue) + } + } else { + t.Fatalf("expected map to have key %s", expectedKey) + } + } + } + + expectedFields := map[string]string{ + "foo": "bar", + "bar": "1", + "baz": "zip", + "frop": "3.14", + } + checkFieldMap(expectedFields, samples[0].Fields) + + expectedFields = map[string]string{ + "foo": "baz", + "bar": "2", + "baz": "zap", + "frop": "4.00", + } + checkFieldMap(expectedFields, samples[1].Fields) +} + +func (u *NoTagsSample) UnmarshalCSVWithFields(key, value string) error { + if u.Fields == nil { + u.Fields = map[string]string{} + } + u.Fields[key] = value + return nil +} + +func TestMultipleStructTags(t *testing.T) { + b := bytes.NewBufferString(`foo,BAR,Baz +e,3,b`) + d := newSimpleDecoderFromReader(b) + + var samples []MultiTagSample + if err := readTo(d, &samples); err != nil { + t.Fatal(err) + } + if samples[0].Foo != "b" { + t.Fatalf("expected second tag value 'b' in multi tag struct field, got %v", samples[0].Foo) + } + + b = bytes.NewBufferString(`foo,BAR +e,3`) + d = newSimpleDecoderFromReader(b) + + if err := readTo(d, &samples); err != nil { + t.Fatal(err) + } + if samples[0].Foo != "e" { + t.Fatalf("wrong value in multi tag struct field, expected 'e', got %v", samples[0].Foo) + } + + b = bytes.NewBufferString(`BAR,Baz +3,b`) + d = newSimpleDecoderFromReader(b) + + if err := readTo(d, &samples); err != nil { + t.Fatal(err) + } + if samples[0].Foo != "b" { + t.Fatal("wrong value in multi tag struct field") + } +} + +func TestStructTagSeparator(t *testing.T) { + b := bytes.NewBufferString(`foo,BAR,Baz +e,3,b`) + d := newSimpleDecoderFromReader(b) + + defaultTagSeparator := TagSeparator + TagSeparator = "|" + defer func() { + TagSeparator = defaultTagSeparator + }() + + var samples []TagSeparatorSample + if err := readTo(d, &samples); err != nil { + t.Fatal(err) + } + + if samples[0].Foo != "b" { + t.Fatal("expected second tag value in multi tag struct field.") + } +} + +func TestCustomTag(t *testing.T) { + b := bytes.NewBufferString(`foo,BAR +e,3`) + d := newSimpleDecoderFromReader(b) + + defaultTagName := TagName + TagName = "custom" + defer func() { + TagName = defaultTagName + }() + + var samples []CustomTagSample + if err := readTo(d, &samples); err != nil { + t.Fatal(err) + } + + if samples[0].Foo != "e" { + t.Fatal("wrong value in custom tag struct field") + } +} + +func TestCSVToMap(t *testing.T) { + b := bytes.NewBufferString(`foo,BAR +4,Jose +2,Daniel +5,Vincent`) + m, err := CSVToMap(bytes.NewReader(b.Bytes())) + if err != nil { + t.Fatal(err) + } + if m["4"] != "Jose" { + t.Fatal("Expected Jose got", m["4"]) + } + if m["2"] != "Daniel" { + t.Fatal("Expected Daniel got", m["2"]) + } + if m["5"] != "Vincent" { + t.Fatal("Expected Vincent got", m["5"]) + } + + b = bytes.NewBufferString(`foo,BAR,Baz +e,3,b`) + _, err = CSVToMap(bytes.NewReader(b.Bytes())) + if err == nil { + t.Fatal("Something went wrong") + } + b = bytes.NewBufferString(`foo +e`) + _, err = CSVToMap(bytes.NewReader(b.Bytes())) + if err == nil { + t.Fatal("Something went wrong") + } +} + +func TestCSVToMaps(t *testing.T) { + defer SetCSVReader(DefaultCSVReader) + type item struct { + setupReader func() + csv *bytes.Buffer + } + data := []item{ + { + setupReader: func() {}, + csv: bytes.NewBufferString(`foo,BAR,Baz +4,Jose,42 +2,Daniel,21 +5,Vincent,84`, + ), + }, + { + setupReader: func() { + SetCSVReader(func(in io.Reader) CSVReader { + r := csv.NewReader(in) + _, err := r.Read() + if err != nil { + t.Fatal(err) + } + return r + }) + }, + csv: bytes.NewBufferString((`,, +foo,BAR,Baz +4,Jose,42 +2,Daniel,21 +5,Vincent,84`), + ), + }, + } + + for _, b := range data { + b.setupReader() + m, err := CSVToMaps(bytes.NewReader(b.csv.Bytes())) + if err != nil { + t.Fatal(err) + } + firstRecord := m[0] + if firstRecord["foo"] != "4" { + t.Fatal("Expected 4 got", firstRecord["foo"]) + } + if firstRecord["BAR"] != "Jose" { + t.Fatal("Expected Jose got", firstRecord["BAR"]) + } + if firstRecord["Baz"] != "42" { + t.Fatal("Expected 42 got", firstRecord["Baz"]) + } + secondRecord := m[1] + if secondRecord["foo"] != "2" { + t.Fatal("Expected 2 got", secondRecord["foo"]) + } + if secondRecord["BAR"] != "Daniel" { + t.Fatal("Expected Daniel got", secondRecord["BAR"]) + } + if secondRecord["Baz"] != "21" { + t.Fatal("Expected 21 got", secondRecord["Baz"]) + } + thirdRecord := m[2] + if thirdRecord["foo"] != "5" { + t.Fatal("Expected 5 got", thirdRecord["foo"]) + } + if thirdRecord["BAR"] != "Vincent" { + t.Fatal("Expected Vincent got", thirdRecord["BAR"]) + } + if thirdRecord["Baz"] != "84" { + t.Fatal("Expected 84 got", thirdRecord["Baz"]) + } + } +} + +type trimDecoder struct { + csvReader CSVReader +} + +func (c *trimDecoder) GetCSVRow() ([]string, error) { + recoder, err := c.csvReader.Read() + for i, r := range recoder { + recoder[i] = strings.TrimRight(r, " ") + } + return recoder, err +} + +func (c *trimDecoder) GetCSVRows() ([][]string, error) { + records := [][]string{} + for { + record, err := c.GetCSVRow() + if err == io.EOF { + break + } else if err != nil { + return nil, err + } + records = append(records, record) + } + return records, nil +} + +func TestUnmarshalToDecoder(t *testing.T) { + blah := 0 + sptr := "*string" + sptr2 := "" + b := bytes.NewBufferString(`foo,BAR,Baz,Blah,SPtr +f,1,baz,, *string +e,3,b,, `) + + var samples []Sample + if err := UnmarshalDecoderToCallback(&trimDecoder{LazyCSVReader(b)}, func(s Sample) { + samples = append(samples, s) + }); err != nil { + t.Fatal(err) + } + if len(samples) != 2 { + t.Fatalf("expected 2 sample instances, got %d", len(samples)) + } + + expected := Sample{Foo: "f", Bar: 1, Baz: "baz", Blah: &blah, SPtr: &sptr} + if !reflect.DeepEqual(expected, samples[0]) { + t.Fatalf("expected first sample %v, got %v", expected, samples[0]) + } + + expected = Sample{Foo: "e", Bar: 3, Baz: "b", Blah: &blah, SPtr: &sptr2} + if !reflect.DeepEqual(expected, samples[1]) { + t.Fatalf("expected second sample %v, got %v", expected, samples[1]) + } +} + +func TestUnmarshalWithoutHeader(t *testing.T) { + blah := 0 + sptr := "" + b := bytes.NewBufferString(`f,1,baz,1.66,,, +e,3,b,,,,`) + d := newSimpleDecoderFromReader(b) + + var samples []Sample + if err := readToWithoutHeaders(d, &samples); err != nil { + t.Fatal(err) + } + + expected := Sample{Foo: "f", Bar: 1, Baz: "baz", Frop: 1.66, Blah: &blah, SPtr: &sptr} + if !reflect.DeepEqual(expected, samples[0]) { + t.Fatalf("expected first sample %v, got %v", expected, samples[0]) + } + + expected = Sample{Foo: "e", Bar: 3, Baz: "b", Frop: 0, Blah: &blah, SPtr: &sptr} + if !reflect.DeepEqual(expected, samples[1]) { + t.Fatalf("expected second sample %v, got %v", expected, samples[1]) + } +} + +func TestUnmarshalCSVWithoutHeaders(t *testing.T) { + // tsv input to test custom csv reader + b := []byte("f\t1\tbaz\ne\t3\tblorp") + r := bytes.NewReader(b) + csvReader := csv.NewReader(r) + csvReader.Comma = '\t' + + var samples []Sample + if err := UnmarshalCSVWithoutHeaders(csvReader, &samples); err != nil { + t.Fatal(err) + } + + expected := Sample{Foo: "f", Bar: 1, Baz: "baz"} + if !reflect.DeepEqual(expected, samples[0]) { + t.Fatalf("expected first sample %v, got %v", expected, samples[0]) + } + expected = Sample{Foo: "e", Bar: 3, Baz: "blorp"} + if !reflect.DeepEqual(expected, samples[1]) { + t.Fatalf("expected second sample %v, got %v", expected, samples[1]) + } +} + +func TestDecodeDefaultValues(t *testing.T) { + type defaultValueStruct struct { + Foo string `csv:"foo,default=x"` + Bar int `csv:"bar,default=42"` + } + b := bytes.NewBufferString(`foo,bar +, +`) + var out []defaultValueStruct + if err := Unmarshal(b, &out); err != nil { + t.Fatal(err) + } + + expected := defaultValueStruct{ + Foo: "x", + Bar: 42, + } + if !reflect.DeepEqual(expected, out[0]) { + t.Fatalf("expected second sample %v, got %v", expected, out[0]) + } +} + +func TestDecodePartialKeys(t *testing.T) { + type defaultValueStruct struct { + Foo string `csv:"foo,partial"` + Bar int `csv:"bar,partial"` + } + b := bytes.NewBufferString(`foolo,ambar +x,42 +`) + var out []defaultValueStruct + if err := Unmarshal(b, &out); err != nil { + t.Fatal(err) + } + + expected := defaultValueStruct{ + Foo: "x", + Bar: 42, + } + if !reflect.DeepEqual(expected, out[0]) { + t.Fatalf("expected second sample %v, got %v", expected, out[0]) + } +} + +func TestTrimTagWhitespace(t *testing.T) { + type whiteSpaceOptionStruct struct { + Foo *string `csv:"foo, omitempty"` + Bar int `csv:"bar, default=13 "` + } + var out []whiteSpaceOptionStruct + b := bytes.NewBufferString(`foo,bar +,`) + if err := Unmarshal(b, &out); err != nil { + t.Fatal(err) + } + expected := whiteSpaceOptionStruct{ + Foo: nil, + Bar: 13, + } + + if !reflect.DeepEqual(expected, out[0]) { + t.Fatalf("expected sample %v, got %v", expected, out[0]) + } +} + +func TestUnmarshalCSVToMap(t *testing.T) { + b := []byte(`line tokens +10 ["PRINT", "\"Hello map!\""] +20 ["GOTO", "10"]`) + r := bytes.NewReader(b) + csvReader := csv.NewReader(r) + csvReader.LazyQuotes = true + csvReader.Comma = '\t' + + var sample map[int][]string + if err := UnmarshalCSVToMap(csvReader, &sample); err != nil { + t.Fatal(err) + } + + expected := map[int][]string{ + 10: {"PRINT", "\"Hello map!\""}, + 20: {"GOTO", "10"}, + } + if !reflect.DeepEqual(expected, sample) { + t.Fatalf("expected %v, got %v", expected, sample) + } +} + +func BenchmarkCSVToMap(b *testing.B) { + bufstring := bytes.NewBufferString(`foo,BAR +4,Jose +2,Daniel +5,Vincent`) + for n := 0; n < b.N; n++ { + _, err := CSVToMap(bytes.NewReader(bufstring.Bytes())) + if err != nil { + b.Fatal(err) + } + } +} + +func BenchmarkUnmarshalCSVToMap(b *testing.B) { + bufstring := []byte(`foo,BAR +4,Jose +2,Daniel +5,Vincent`) + for n := 0; n < b.N; n++ { + var sample map[string]string + r := bytes.NewReader(bufstring) + d := csv.NewReader(r) + err := UnmarshalCSVToMap(d, &sample) + if err != nil { + b.Fatal(err) + } + } +} + +func Test_readTo_nested_struct(t *testing.T) { + b := bytes.NewBufferString(`one.boolField1,one.stringField2,two.boolField1,two.stringField2,three.boolField1,three.stringField2 +false,email_one,true,email_two,false,email_three`) + d := newSimpleDecoderFromReader(b) + + var samples []NestedSample + err := readTo(d, &samples) + if err != nil { + t.Fatalf("failed to unmarshal: %v", err) + } + + expected := []NestedSample{ + { + Inner1: InnerStruct{ + BoolIgnoreField0: false, + BoolField1: false, + StringField2: "email_one", + }, + Inner2: InnerStruct{ + BoolIgnoreField0: false, + BoolField1: true, + StringField2: "email_two", + }, + Inner3: NestedEmbedSample{InnerStruct{ + BoolIgnoreField0: false, + BoolField1: false, + StringField2: "email_three", + }}, + }, + } + + if !reflect.DeepEqual(expected, samples) { + t.Fatalf("expected \n sample: %v\n got: %v", expected, samples) + } +} + +func Test_readTo_inline_nested_struct(t *testing.T) { + b := bytes.NewBufferString("a,b,x\n1,2,3") + d := newSimpleDecoderFromReader(b) + + var samples []InlineFooSample + if err := readTo(d, &samples); err != nil { + t.Fatalf("failed to unmarshal: %v", err) + } + + expected := []InlineFooSample{{Bar: InlineBar{A: 1, B: 2}, X: 3}} + if !reflect.DeepEqual(expected, samples) { + t.Fatalf("expected \n sample: %v\n got: %v", expected, samples) + } +} diff --git a/types.go b/types.go new file mode 100644 index 0000000..761208e --- /dev/null +++ b/types.go @@ -0,0 +1,493 @@ +package gocsv + +import ( + "encoding" + "fmt" + "reflect" + "strconv" + "strings" + + "encoding/json" +) + +// -------------------------------------------------------------------------- +// Conversion interfaces + +var ( + marshalerType = reflect.TypeOf(new(TypeMarshaller)).Elem() + textMarshalerType = reflect.TypeOf(new(encoding.TextMarshaler)).Elem() + unmarshalerType = reflect.TypeOf(new(TypeUnmarshaller)).Elem() + unmarshalCSVWithFieldsType = reflect.TypeOf(new(TypeUnmarshalCSVWithFields)).Elem() +) + +// TypeMarshaller is implemented by any value that has a MarshalCSV method +// This converter is used to convert the value to it string representation +type TypeMarshaller interface { + MarshalCSV() (string, error) +} + +// TypeUnmarshaller is implemented by any value that has an UnmarshalCSV method +// This converter is used to convert a string to your value representation of that string +type TypeUnmarshaller interface { + UnmarshalCSV(string) error +} + +// TypeUnmarshalCSVWithFields can be implemented on whole structs to allow for whole structures to customized internal vs one off fields +type TypeUnmarshalCSVWithFields interface { + UnmarshalCSVWithFields(key, value string) error +} + +// NoUnmarshalFuncError is the custom error type to be raised in case there is no unmarshal function defined on type +type NoUnmarshalFuncError struct { + t reflect.Type +} + +func (e NoUnmarshalFuncError) Error() string { + return "No known conversion from string to " + e.t.Name() + ", it does not implement TypeUnmarshaller" +} + +// NoMarshalFuncError is the custom error type to be raised in case there is no marshal function defined on type +type NoMarshalFuncError struct { + ty reflect.Type +} + +func (e NoMarshalFuncError) Error() string { + return "No known conversion from " + e.ty.String() + " to string, " + e.ty.String() + " does not implement TypeMarshaller nor Stringer" +} + +// -------------------------------------------------------------------------- +// Conversion helpers + +func toString(in interface{}) (string, error) { + inValue := reflect.ValueOf(in) + + switch inValue.Kind() { + case reflect.String: + return inValue.String(), nil + case reflect.Bool: + b := inValue.Bool() + if b { + return "true", nil + } + return "false", nil + case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64: + return fmt.Sprintf("%v", inValue.Int()), nil + case reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64: + return fmt.Sprintf("%v", inValue.Uint()), nil + case reflect.Float32: + return strconv.FormatFloat(inValue.Float(), byte('f'), -1, 32), nil + case reflect.Float64: + return strconv.FormatFloat(inValue.Float(), byte('f'), -1, 64), nil + } + return "", fmt.Errorf("No known conversion from " + inValue.Type().String() + " to string") +} + +func toBool(in interface{}) (bool, error) { + inValue := reflect.ValueOf(in) + + switch inValue.Kind() { + case reflect.String: + s := inValue.String() + s = strings.TrimSpace(s) + if strings.EqualFold(s, "yes") { + return true, nil + } else if strings.EqualFold(s, "no") || s == "" { + return false, nil + } else { + return strconv.ParseBool(s) + } + case reflect.Bool: + return inValue.Bool(), nil + case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64: + i := inValue.Int() + if i != 0 { + return true, nil + } + return false, nil + case reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64: + i := inValue.Uint() + if i != 0 { + return true, nil + } + return false, nil + case reflect.Float32, reflect.Float64: + f := inValue.Float() + if f != 0 { + return true, nil + } + return false, nil + } + return false, fmt.Errorf("No known conversion from " + inValue.Type().String() + " to bool") +} + +func toInt(in interface{}) (int64, error) { + inValue := reflect.ValueOf(in) + + switch inValue.Kind() { + case reflect.String: + s := strings.TrimSpace(inValue.String()) + if s == "" { + return 0, nil + } + out := strings.SplitN(s, ".", 2) + return strconv.ParseInt(out[0], 0, 64) + case reflect.Bool: + if inValue.Bool() { + return 1, nil + } + return 0, nil + case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64: + return inValue.Int(), nil + case reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64: + return int64(inValue.Uint()), nil + case reflect.Float32, reflect.Float64: + return int64(inValue.Float()), nil + } + return 0, fmt.Errorf("No known conversion from " + inValue.Type().String() + " to int") +} + +func toUint(in interface{}) (uint64, error) { + inValue := reflect.ValueOf(in) + + switch inValue.Kind() { + case reflect.String: + s := strings.TrimSpace(inValue.String()) + if s == "" { + return 0, nil + } + + // support the float input + if strings.Contains(s, ".") { + f, err := strconv.ParseFloat(s, 64) + if err != nil { + return 0, err + } + return uint64(f), nil + } + return strconv.ParseUint(s, 0, 64) + case reflect.Bool: + if inValue.Bool() { + return 1, nil + } + return 0, nil + case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64: + return uint64(inValue.Int()), nil + case reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64: + return inValue.Uint(), nil + case reflect.Float32, reflect.Float64: + return uint64(inValue.Float()), nil + } + return 0, fmt.Errorf("No known conversion from " + inValue.Type().String() + " to uint") +} + +func toFloat(in interface{}) (float64, error) { + inValue := reflect.ValueOf(in) + + switch inValue.Kind() { + case reflect.String: + s := strings.TrimSpace(inValue.String()) + if s == "" { + return 0, nil + } + s = strings.Replace(s, ",", ".", -1) + return strconv.ParseFloat(s, 64) + case reflect.Bool: + if inValue.Bool() { + return 1, nil + } + return 0, nil + case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64: + return float64(inValue.Int()), nil + case reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64: + return float64(inValue.Uint()), nil + case reflect.Float32, reflect.Float64: + return inValue.Float(), nil + } + return 0, fmt.Errorf("No known conversion from " + inValue.Type().String() + " to float") +} + +func setField(field reflect.Value, value string, omitEmpty bool) error { + if field.Kind() == reflect.Ptr { + if omitEmpty && value == "" { + return nil + } + if field.IsNil() { + field.Set(reflect.New(field.Type().Elem())) + } + field = field.Elem() + } + + switch field.Interface().(type) { + case string: + s, err := toString(value) + if err != nil { + return err + } + field.SetString(s) + case bool: + b, err := toBool(value) + if err != nil { + return err + } + field.SetBool(b) + case int, int8, int16, int32, int64: + i, err := toInt(value) + if err != nil { + return err + } + field.SetInt(i) + case uint, uint8, uint16, uint32, uint64: + ui, err := toUint(value) + if err != nil { + return err + } + field.SetUint(ui) + case float32, float64: + f, err := toFloat(value) + if err != nil { + return err + } + field.SetFloat(f) + default: + // Not a native type, check for unmarshal method + if err := unmarshall(field, value); err != nil { + if _, ok := err.(NoUnmarshalFuncError); !ok { + return err + } + // Could not unmarshal, check for kind, e.g. renamed type from basic type + switch field.Kind() { + case reflect.String: + s, err := toString(value) + if err != nil { + return err + } + field.SetString(s) + case reflect.Bool: + b, err := toBool(value) + if err != nil { + return err + } + field.SetBool(b) + case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64: + i, err := toInt(value) + if err != nil { + return err + } + field.SetInt(i) + case reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64: + ui, err := toUint(value) + if err != nil { + return err + } + field.SetUint(ui) + case reflect.Float32, reflect.Float64: + f, err := toFloat(value) + if err != nil { + return err + } + field.SetFloat(f) + case reflect.Slice, reflect.Struct: + if value == "" { + return nil + } + + err := json.Unmarshal([]byte(value), field.Addr().Interface()) + if err != nil { + return err + } + default: + return err + } + } else { + return nil + } + } + return nil +} + +func getFieldAsString(field reflect.Value) (str string, err error) { + switch field.Kind() { + case reflect.Interface, reflect.Ptr: + if field.IsNil() { + return "", nil + } + return getFieldAsString(field.Elem()) + default: + // Check if field is go native type + switch field.Interface().(type) { + case string: + return field.String(), nil + case bool: + if field.Bool() { + return "true", nil + } else { + return "false", nil + } + case int, int8, int16, int32, int64: + return fmt.Sprintf("%v", field.Int()), nil + case uint, uint8, uint16, uint32, uint64: + return fmt.Sprintf("%v", field.Uint()), nil + case float32: + str, err = toString(float32(field.Float())) + if err != nil { + return str, err + } + case float64: + str, err = toString(field.Float()) + if err != nil { + return str, err + } + default: + // Not a native type, check for marshal method + str, err = marshall(field) + if err != nil { + if _, ok := err.(NoMarshalFuncError); !ok { + return str, err + } + // If not marshal method, is field compatible with/renamed from native type + switch field.Kind() { + case reflect.String: + return field.String(), nil + case reflect.Bool: + str, err = toString(field.Bool()) + if err != nil { + return str, err + } + case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64: + str, err = toString(field.Int()) + if err != nil { + return str, err + } + case reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64: + str, err = toString(field.Uint()) + if err != nil { + return str, err + } + case reflect.Float32: + str, err = toString(float32(field.Float())) + if err != nil { + return str, err + } + case reflect.Float64: + str, err = toString(field.Float()) + if err != nil { + return str, err + } + case reflect.Slice: + fallthrough + case reflect.Array: + value := field.Interface() + if field.CanAddr() { + value = field.Addr().Interface() + } + b, err := json.Marshal(value) + if err != nil { + return str, err + } + + str = string(b) + } + } else { + return str, nil + } + } + } + return str, nil +} + +// -------------------------------------------------------------------------- +// Un/serializations helpers + +func canMarshal(t reflect.Type) bool { + // Struct that implements any of the text or CSV marshaling interfaces + if t.Implements(marshalerType) || + t.Implements(textMarshalerType) || + t.Implements(unmarshalerType) || + t.Implements(unmarshalCSVWithFieldsType) { + return true + } + + // Pointer to a struct that implements any of the text or CSV marshaling interfaces + t = reflect.PtrTo(t) + if t.Implements(marshalerType) || + t.Implements(textMarshalerType) || + t.Implements(unmarshalerType) || + t.Implements(unmarshalCSVWithFieldsType) { + return true + } + return false +} + +func unmarshall(field reflect.Value, value string) error { + dupField := field + unMarshallIt := func(finalField reflect.Value) error { + if finalField.CanInterface() { + fieldIface := finalField.Interface() + + fieldTypeUnmarshaller, ok := fieldIface.(TypeUnmarshaller) + if ok { + return fieldTypeUnmarshaller.UnmarshalCSV(value) + } + + // Otherwise try to use TextUnmarshaler + fieldTextUnmarshaler, ok := fieldIface.(encoding.TextUnmarshaler) + if ok { + return fieldTextUnmarshaler.UnmarshalText([]byte(value)) + } + } + + return NoUnmarshalFuncError{field.Type()} + } + for dupField.Kind() == reflect.Interface || dupField.Kind() == reflect.Ptr { + if dupField.IsNil() { + dupField = reflect.New(field.Type().Elem()) + field.Set(dupField) + return unMarshallIt(dupField) + } + dupField = dupField.Elem() + } + if dupField.CanAddr() { + return unMarshallIt(dupField.Addr()) + } + return NoUnmarshalFuncError{field.Type()} +} + +func marshall(field reflect.Value) (value string, err error) { + dupField := field + marshallIt := func(finalField reflect.Value) (string, error) { + if finalField.CanInterface() { + fieldIface := finalField.Interface() + + // Use TypeMarshaller when possible + fieldTypeMarhaller, ok := fieldIface.(TypeMarshaller) + if ok { + return fieldTypeMarhaller.MarshalCSV() + } + + // Otherwise try to use TextMarshaller + fieldTextMarshaler, ok := fieldIface.(encoding.TextMarshaler) + if ok { + text, err := fieldTextMarshaler.MarshalText() + return string(text), err + } + + // Otherwise try to use Stringer + fieldStringer, ok := fieldIface.(fmt.Stringer) + if ok { + return fieldStringer.String(), nil + } + } + + return value, NoMarshalFuncError{field.Type()} + } + for dupField.Kind() == reflect.Interface || dupField.Kind() == reflect.Ptr { + if dupField.IsNil() { + return value, nil + } + dupField = dupField.Elem() + } + if dupField.CanAddr() { + dupField = dupField.Addr() + } + return marshallIt(dupField) +} diff --git a/types_test.go b/types_test.go new file mode 100644 index 0000000..3c85c0a --- /dev/null +++ b/types_test.go @@ -0,0 +1,141 @@ +package gocsv + +import ( + "reflect" + "testing" +) + +type sampleTypeUnmarshaller struct { + val string +} + +func (s *sampleTypeUnmarshaller) UnmarshalCSV(val string) error { + s.val = val + return nil +} + +func (s sampleTypeUnmarshaller) MarshalCSV() (string, error) { + return s.val, nil +} + +type sampleTextUnmarshaller struct { + val []byte +} + +func (s *sampleTextUnmarshaller) UnmarshalText(text []byte) error { + s.val = text + return nil +} + +func (s sampleTextUnmarshaller) MarshalText() ([]byte, error) { + return s.val, nil +} + +type sampleStringer string + +func (s sampleStringer) String() string { + return string(s) +} + +type stringAlias string +type customStringAlias string + +func (s customStringAlias) MarshalCSV() (string, error) { + return `"` + string(s) + `"`, nil +} + +func Test_getFieldAsString_CustomStringAlias(t *testing.T) { + s, err := getFieldAsString(reflect.ValueOf(customStringAlias("foo"))) + if err != nil { + t.Fatalf("getFieldAsString failure: %s", err) + } + + if string(s) != `"foo"` { + t.Fatalf(`expected "foo" got %s`, s) + } + + s, err = getFieldAsString(reflect.ValueOf(stringAlias("foo"))) + if err != nil { + t.Fatalf("getFieldAsString failure: %s", err) + } + + if string(s) != `foo` { + t.Fatalf(`expected foo got %s`, s) + } +} + +func Benchmark_unmarshall_TypeUnmarshaller(b *testing.B) { + sample := sampleTypeUnmarshaller{} + val := reflect.ValueOf(&sample) + for n := 0; n < b.N; n++ { + if err := unmarshall(val, "foo"); err != nil { + b.Fatalf("unmarshall error: %s", err.Error()) + } + } +} + +func Benchmark_unmarshall_TextUnmarshaller(b *testing.B) { + sample := sampleTextUnmarshaller{} + val := reflect.ValueOf(&sample) + for n := 0; n < b.N; n++ { + if err := unmarshall(val, "foo"); err != nil { + b.Fatalf("unmarshall error: %s", err.Error()) + } + } +} + +func Benchmark_marshall_TypeMarshaller(b *testing.B) { + sample := sampleTypeUnmarshaller{"foo"} + val := reflect.ValueOf(&sample) + for n := 0; n < b.N; n++ { + _, err := marshall(val) + if err != nil { + b.Fatalf("marshall error: %s", err.Error()) + } + } +} + +func Benchmark_marshall_TextMarshaller(b *testing.B) { + sample := sampleTextUnmarshaller{[]byte("foo")} + val := reflect.ValueOf(&sample) + for n := 0; n < b.N; n++ { + _, err := marshall(val) + if err != nil { + b.Fatalf("marshall error: %s", err.Error()) + } + } +} + +func Benchmark_marshall_Stringer(b *testing.B) { + sample := sampleStringer("foo") + val := reflect.ValueOf(&sample) + for n := 0; n < b.N; n++ { + _, err := marshall(val) + if err != nil { + b.Fatalf("marshall error: %s", err.Error()) + } + } +} + +func TestToInt(t *testing.T) { + TestCase := []struct { + field string + result int + err error + }{ + {"123.2", 123, nil}, + {"123", 123, nil}, + {"1.2.3", 1, nil}, + {"0.123", 0, nil}, + } + + for idx, item := range TestCase { + out, err := toInt(item.field) + if err != nil { + t.Fatal(err) + } + if int(out) != item.result { + t.Fatal(idx, "result not equal") + } + } +}