diff --git a/csv.go b/csv.go new file mode 100644 index 0000000..aeb6f85 --- /dev/null +++ b/csv.go @@ -0,0 +1,566 @@ +// Copyright 2014 Jonathan Picques. All rights reserved. +// Use of this source code is governed by a MIT license +// The license can be found in the LICENSE file. + +// The GoCSV package aims to provide easy CSV serialization and deserialization to the golang programming language + +package gocsv + +import ( + "bytes" + "encoding/csv" + "fmt" + "io" + "mime/multipart" + "os" + "reflect" + "strings" + "sync" +) + +// FailIfUnmatchedStructTags indicates whether it is considered an error when there is an unmatched +// struct tag. +var FailIfUnmatchedStructTags = false + +// FailIfDoubleHeaderNames indicates whether it is considered an error when a header name is repeated +// in the csv header. +var FailIfDoubleHeaderNames = false + +// ShouldAlignDuplicateHeadersWithStructFieldOrder indicates whether we should align duplicate CSV +// headers per their alignment in the struct definition. +var ShouldAlignDuplicateHeadersWithStructFieldOrder = false + +// TagName defines key in the struct field's tag to scan +var TagName = "csv" + +// TagSeparator defines seperator string for multiple csv tags in struct fields +var TagSeparator = "," + +// FieldSeperator defines how to combine parent struct with child struct +var FieldsCombiner = "." + +// Normalizer is a function that takes and returns a string. It is applied to +// struct and header field values before they are compared. It can be used to alter +// names for comparison. For instance, you could allow case insensitive matching +// or convert '-' to '_'. +type Normalizer func(string) string + +type ErrorHandler func(*csv.ParseError) bool + +// normalizeName function initially set to a nop Normalizer. +var normalizeName = DefaultNameNormalizer() + +// DefaultNameNormalizer is a nop Normalizer. +func DefaultNameNormalizer() Normalizer { return func(s string) string { return s } } + +// SetHeaderNormalizer sets the normalizer used to normalize struct and header field names. +func SetHeaderNormalizer(f Normalizer) { + normalizeName = f + // Need to clear the cache hen the header normalizer changes. + structInfoCache = sync.Map{} +} + +// -------------------------------------------------------------------------- +// CSVWriter used to format CSV + +var selfCSVWriter = DefaultCSVWriter + +// DefaultCSVWriter is the default SafeCSVWriter used to format CSV (cf. csv.NewWriter) +func DefaultCSVWriter(out io.Writer) *SafeCSVWriter { + writer := NewSafeCSVWriter(csv.NewWriter(out)) + + // As only one rune can be defined as a CSV separator, we are going to trim + // the custom tag separator and use the first rune. + if runes := []rune(strings.TrimSpace(TagSeparator)); len(runes) > 0 { + writer.Comma = runes[0] + } + + return writer +} + +// SetCSVWriter sets the SafeCSVWriter used to format CSV. +func SetCSVWriter(csvWriter func(io.Writer) *SafeCSVWriter) { + selfCSVWriter = csvWriter +} + +func getCSVWriter(out io.Writer) *SafeCSVWriter { + return selfCSVWriter(out) +} + +// -------------------------------------------------------------------------- +// CSVReader used to parse CSV + +var selfCSVReader = DefaultCSVReader + +// DefaultCSVReader is the default CSV reader used to parse CSV (cf. csv.NewReader) +func DefaultCSVReader(in io.Reader) CSVReader { + return csv.NewReader(in) +} + +// LazyCSVReader returns a lazy CSV reader, with LazyQuotes and TrimLeadingSpace. +func LazyCSVReader(in io.Reader) CSVReader { + csvReader := csv.NewReader(in) + csvReader.LazyQuotes = true + csvReader.TrimLeadingSpace = true + return csvReader +} + +// SetCSVReader sets the CSV reader used to parse CSV. +func SetCSVReader(csvReader func(io.Reader) CSVReader) { + selfCSVReader = csvReader +} + +func getCSVReader(in io.Reader) CSVReader { + return selfCSVReader(in) +} + +// -------------------------------------------------------------------------- +// Marshal functions + +// MarshalFile saves the interface as CSV in the file. +func MarshalFile(in interface{}, file *os.File) (err error) { + return Marshal(in, file) +} + +// MarshalString returns the CSV string from the interface. +func MarshalString(in interface{}) (out string, err error) { + bufferString := bytes.NewBufferString(out) + if err := Marshal(in, bufferString); err != nil { + return "", err + } + return bufferString.String(), nil +} + +// MarshalStringWithoutHeaders returns the CSV string from the interface. +func MarshalStringWithoutHeaders(in interface{}) (out string, err error) { + bufferString := bytes.NewBufferString(out) + if err := MarshalWithoutHeaders(in, bufferString); err != nil { + return "", err + } + return bufferString.String(), nil +} + +// MarshalBytes returns the CSV bytes from the interface. +func MarshalBytes(in interface{}) (out []byte, err error) { + bufferString := bytes.NewBuffer(out) + if err := Marshal(in, bufferString); err != nil { + return nil, err + } + return bufferString.Bytes(), nil +} + +// Marshal returns the CSV in writer from the interface. +func Marshal(in interface{}, out io.Writer) (err error) { + writer := getCSVWriter(out) + return writeTo(writer, in, false) +} + +// MarshalWithoutHeaders returns the CSV in writer from the interface. +func MarshalWithoutHeaders(in interface{}, out io.Writer) (err error) { + writer := getCSVWriter(out) + return writeTo(writer, in, true) +} + +// MarshalChan returns the CSV read from the channel. +func MarshalChan(c <-chan interface{}, out CSVWriter) error { + return writeFromChan(out, c, false) +} + +// MarshalChanWithoutHeaders returns the CSV read from the channel. +func MarshalChanWithoutHeaders(c <-chan interface{}, out CSVWriter) error { + return writeFromChan(out, c, true) +} + +// MarshalCSV returns the CSV in writer from the interface. +func MarshalCSV(in interface{}, out CSVWriter) (err error) { + return writeTo(out, in, false) +} + +// MarshalCSVWithoutHeaders returns the CSV in writer from the interface. +func MarshalCSVWithoutHeaders(in interface{}, out CSVWriter) (err error) { + return writeTo(out, in, true) +} + +// -------------------------------------------------------------------------- +// Unmarshal functions + +// UnmarshalFile parses the CSV from the file in the interface. +func UnmarshalFile(in *os.File, out interface{}) error { + return Unmarshal(in, out) +} + +// UnmarshalMultipartFile parses the CSV from the multipart file in the interface. +func UnmarshalMultipartFile(in *multipart.File, out interface{}) error { + return Unmarshal(convertTo(in), out) +} + +// UnmarshalFileWithErrorHandler parses the CSV from the file in the interface. +func UnmarshalFileWithErrorHandler(in *os.File, errHandler ErrorHandler, out interface{}) error { + return UnmarshalWithErrorHandler(in, errHandler, out) +} + +// UnmarshalString parses the CSV from the string in the interface. +func UnmarshalString(in string, out interface{}) error { + return Unmarshal(strings.NewReader(in), out) +} + +// UnmarshalBytes parses the CSV from the bytes in the interface. +func UnmarshalBytes(in []byte, out interface{}) error { + return Unmarshal(bytes.NewReader(in), out) +} + +// Unmarshal parses the CSV from the reader in the interface. +func Unmarshal(in io.Reader, out interface{}) error { + return readTo(newSimpleDecoderFromReader(in), out) +} + +// Unmarshal parses the CSV from the reader in the interface. +func UnmarshalWithErrorHandler(in io.Reader, errHandle ErrorHandler, out interface{}) error { + return readToWithErrorHandler(newSimpleDecoderFromReader(in), errHandle, out) +} + +// UnmarshalWithoutHeaders parses the CSV from the reader in the interface. +func UnmarshalWithoutHeaders(in io.Reader, out interface{}) error { + return readToWithoutHeaders(newSimpleDecoderFromReader(in), out) +} + +// UnmarshalCSVWithoutHeaders parses a headerless CSV with passed in CSV reader +func UnmarshalCSVWithoutHeaders(in CSVReader, out interface{}) error { + return readToWithoutHeaders(csvDecoder{in}, out) +} + +// UnmarshalDecoder parses the CSV from the decoder in the interface +func UnmarshalDecoder(in Decoder, out interface{}) error { + return readTo(in, out) +} + +// UnmarshalCSV parses the CSV from the reader in the interface. +func UnmarshalCSV(in CSVReader, out interface{}) error { + return readTo(csvDecoder{in}, out) +} + +// UnmarshalCSVToMap parses a CSV of 2 columns into a map. +func UnmarshalCSVToMap(in CSVReader, out interface{}) error { + decoder := NewSimpleDecoderFromCSVReader(in) + header, err := decoder.GetCSVRow() + if err != nil { + return err + } + if len(header) != 2 { + return fmt.Errorf("maps can only be created for csv of two columns") + } + outValue, outType := getConcreteReflectValueAndType(out) + if outType.Kind() != reflect.Map { + return fmt.Errorf("cannot use " + outType.String() + ", only map supported") + } + keyType := outType.Key() + valueType := outType.Elem() + outValue.Set(reflect.MakeMap(outType)) + for { + key := reflect.New(keyType) + value := reflect.New(valueType) + line, err := decoder.GetCSVRow() + if err == io.EOF { + break + } else if err != nil { + return err + } + if err := setField(key, line[0], false); err != nil { + return err + } + if err := setField(value, line[1], false); err != nil { + return err + } + outValue.SetMapIndex(key.Elem(), value.Elem()) + } + return nil +} + +// UnmarshalToChan parses the CSV from the reader and send each value in the chan c. +// The channel must have a concrete type. +func UnmarshalToChan(in io.Reader, c interface{}) error { + if c == nil { + return fmt.Errorf("goscv: channel is %v", c) + } + return readEach(newSimpleDecoderFromReader(in), nil, c) +} + +// UnmarshalToChanWithErrorHandler parses the CSV from the reader in the interface. +func UnmarshalToChanWithErrorHandler(in io.Reader, errorHandler ErrorHandler, c interface{}) error { + if c == nil { + return fmt.Errorf("goscv: channel is %v", c) + } + return readEach(newSimpleDecoderFromReader(in), errorHandler, c) +} + +// UnmarshalToChanWithoutHeaders parses the CSV from the reader and send each value in the chan c. +// The channel must have a concrete type. +func UnmarshalToChanWithoutHeaders(in io.Reader, c interface{}) error { + if c == nil { + return fmt.Errorf("goscv: channel is %v", c) + } + return readEachWithoutHeaders(newSimpleDecoderFromReader(in), c) +} + +// UnmarshalDecoderToChan parses the CSV from the decoder and send each value in the chan c. +// The channel must have a concrete type. +func UnmarshalDecoderToChan(in SimpleDecoder, c interface{}) error { + if c == nil { + return fmt.Errorf("goscv: channel is %v", c) + } + return readEach(in, nil, c) +} + +// UnmarshalStringToChan parses the CSV from the string and send each value in the chan c. +// The channel must have a concrete type. +func UnmarshalStringToChan(in string, c interface{}) error { + return UnmarshalToChan(strings.NewReader(in), c) +} + +// UnmarshalBytesToChan parses the CSV from the bytes and send each value in the chan c. +// The channel must have a concrete type. +func UnmarshalBytesToChan(in []byte, c interface{}) error { + return UnmarshalToChan(bytes.NewReader(in), c) +} + +// UnmarshalToCallback parses the CSV from the reader and send each value to the given func f. +// The func must look like func(Struct). If its last return value is an error, +// the parser is drained without further callback calls before returning that error. +func UnmarshalToCallback(in io.Reader, f interface{}) error { + valueFunc := reflect.ValueOf(f) + t := reflect.TypeOf(f) + if t.NumIn() != 1 { + return fmt.Errorf("the given function must have exactly one parameter") + } + cerr := make(chan error) + c := reflect.MakeChan(reflect.ChanOf(reflect.BothDir, t.In(0)), 0) + go func() { + cerr <- UnmarshalToChan(in, c.Interface()) + }() + var callbackErr error + for { + select { + case err := <-cerr: + if callbackErr != nil { + return callbackErr + } + return err + default: + } + v, notClosed := c.Recv() + if !notClosed || v.Interface() == nil { + break + } + if callbackErr != nil { + continue // Drain the producer without invoking the callback again. + } + callResults := valueFunc.Call([]reflect.Value{v}) + // Preserve a callback error while allowing the producer to finish. + if len(callResults) > 0 { + if err, ok := callResults[len(callResults)-1].Interface().(error); ok { + callbackErr = err + } + } + } + err := <-cerr + if callbackErr != nil { + return callbackErr + } + return err +} + +// UnmarshalDecoderToCallback parses the CSV from the decoder and send each value to the given func f. +// The func must look like func(Struct). +func UnmarshalDecoderToCallback(in SimpleDecoder, f interface{}) error { + valueFunc := reflect.ValueOf(f) + t := reflect.TypeOf(f) + if t.NumIn() != 1 { + return fmt.Errorf("the given function must have exactly one parameter") + } + cerr := make(chan error) + c := reflect.MakeChan(reflect.ChanOf(reflect.BothDir, t.In(0)), 0) + go func() { + cerr <- UnmarshalDecoderToChan(in, c.Interface()) + }() + for { + select { + case err := <-cerr: + return err + default: + } + v, notClosed := c.Recv() + if !notClosed || v.Interface() == nil { + break + } + valueFunc.Call([]reflect.Value{v}) + } + return <-cerr +} + +// UnmarshalBytesToCallback parses the CSV from the bytes and send each value to the given func f. +// The func must look like func(Struct). +func UnmarshalBytesToCallback(in []byte, f interface{}) error { + return UnmarshalToCallback(bytes.NewReader(in), f) +} + +// UnmarshalStringToCallback parses the CSV from the string and send each value to the given func f. +// The func must look like func(Struct). +func UnmarshalStringToCallback(in string, c interface{}) (err error) { + return UnmarshalToCallback(strings.NewReader(in), c) +} + +// UnmarshalToCallbackWithError parses the CSV from the reader and +// send each value to the given func f. +// +// If func returns error, it will stop processing, drain the +// parser and propagate the error to caller. +// +// The func must look like func(Struct) error. +func UnmarshalToCallbackWithError(in io.Reader, f interface{}) error { + valueFunc := reflect.ValueOf(f) + t := reflect.TypeOf(f) + if t.NumIn() != 1 { + return fmt.Errorf("the given function must have exactly one parameter") + } + if t.NumOut() != 1 { + return fmt.Errorf("the given function must have exactly one return value") + } + if !isErrorType(t.Out(0)) { + return fmt.Errorf("the given function must only return error") + } + + cerr := make(chan error) + c := reflect.MakeChan(reflect.ChanOf(reflect.BothDir, t.In(0)), 0) + go func() { + cerr <- UnmarshalToChan(in, c.Interface()) + }() + + var fErr error + for { + select { + case err := <-cerr: + if err != nil { + return err + } + return fErr + default: + } + v, notClosed := c.Recv() + if !notClosed || v.Interface() == nil { + if err := <-cerr; err != nil { + fErr = err + } + break + } + + // callback f has already returned an error, stop processing but keep draining the chan c + if fErr != nil { + continue + } + + results := valueFunc.Call([]reflect.Value{v}) + + // If the callback f returns an error, stores it and returns it in future. + errValue := results[0].Interface() + if errValue != nil { + fErr = errValue.(error) + } + } + return fErr +} + +// UnmarshalBytesToCallbackWithError parses the CSV from the bytes and +// send each value to the given func f. +// +// If func returns error, it will stop processing, drain the +// parser and propagate the error to caller. +// +// The func must look like func(Struct) error. +func UnmarshalBytesToCallbackWithError(in []byte, f interface{}) error { + return UnmarshalToCallbackWithError(bytes.NewReader(in), f) +} + +// UnmarshalStringToCallbackWithError parses the CSV from the string and +// send each value to the given func f. +// +// If func returns error, it will stop processing, drain the +// parser and propagate the error to caller. +// +// The func must look like func(Struct) error. +func UnmarshalStringToCallbackWithError(in string, c interface{}) (err error) { + return UnmarshalToCallbackWithError(strings.NewReader(in), c) +} + +// CSVToMap creates a simple map from a CSV of 2 columns. +func CSVToMap(in io.Reader) (map[string]string, error) { + decoder := newSimpleDecoderFromReader(in) + header, err := decoder.GetCSVRow() + if err != nil { + return nil, err + } + if len(header) != 2 { + return nil, fmt.Errorf("maps can only be created for csv of two columns") + } + m := make(map[string]string) + for { + line, err := decoder.GetCSVRow() + if err == io.EOF { + break + } else if err != nil { + return nil, err + } + m[line[0]] = line[1] + } + return m, nil +} + +// CSVToMaps takes a reader and returns an array of dictionaries, using the header row as the keys +func CSVToMaps(reader io.Reader) ([]map[string]string, error) { + r := getCSVReader(reader) + rows := []map[string]string{} + var header []string + for { + record, err := r.Read() + if err == io.EOF { + break + } + if err != nil { + return nil, err + } + if header == nil { + header = record + } else { + dict := map[string]string{} + for i := range header { + dict[header[i]] = record[i] + } + rows = append(rows, dict) + } + } + return rows, nil +} + +// CSVToChanMaps parses the CSV from the reader and send a dictionary in the chan c, using the header row as the keys. +func CSVToChanMaps(reader io.Reader, c chan<- map[string]string) error { + r := csv.NewReader(reader) + var header []string + for { + record, err := r.Read() + if err == io.EOF { + break + } + if err != nil { + return err + } + if header == nil { + header = record + } else { + dict := map[string]string{} + for i := range header { + dict[header[i]] = record[i] + } + c <- dict + } + } + return nil +} diff --git a/encode_test.go b/encode_test.go new file mode 100644 index 0000000..f80af84 --- /dev/null +++ b/encode_test.go @@ -0,0 +1,728 @@ +package gocsv + +import ( + "bytes" + "encoding/csv" + "errors" + "io" + "io/ioutil" + "math" + "strconv" + "strings" + "testing" + "time" +) + +func assertLine(t *testing.T, expected, actual []string) { + if len(expected) != len(actual) { + t.Fatalf("line length mismatch between expected: %d and actual: %d\nExpected:\n%v\nActual:\n%v\n", len(expected), len(actual), expected, actual) + } + for i := range expected { + if expected[i] != actual[i] { + t.Fatalf("mismatch on field %d at line `%s`: %s != %s", i, expected, expected[i], actual[i]) + } + } +} + +func Test_writeTo(t *testing.T) { + b := bytes.Buffer{} + e := &encoder{out: &b} + blah := 2 + sptr := "*string" + s := []Sample{ + {Foo: "f", Bar: 1, Baz: "baz", Frop: 0.1, Blah: &blah, SPtr: &sptr}, + {Foo: "e", Bar: 3, Baz: "b", Frop: 6.0 / 13, Blah: nil, SPtr: nil}, + } + if err := writeTo(NewSafeCSVWriter(csv.NewWriter(e.out)), s, false); err != nil { + t.Fatal(err) + } + + lines, err := csv.NewReader(&b).ReadAll() + if err != nil { + t.Fatal(err) + } + if len(lines) != 3 { + t.Fatalf("expected 3 lines, got %d", len(lines)) + } + assertLine(t, []string{"foo", "BAR", "Baz", "Quux", "Blah", "SPtr", "Omit"}, lines[0]) + assertLine(t, []string{"f", "1", "baz", "0.1", "2", "*string", ""}, lines[1]) + assertLine(t, []string{"e", "3", "b", "0.46153846153846156", "", "", ""}, lines[2]) +} + +func Test_writeTo_Time(t *testing.T) { + b := bytes.Buffer{} + e := &encoder{out: &b} + d := time.Unix(60, 0) + s := []DateTime{ + {Foo: d}, + } + if err := writeTo(NewSafeCSVWriter(csv.NewWriter(e.out)), s, true); err != nil { + t.Fatal(err) + } + + lines, err := csv.NewReader(&b).ReadAll() + if err != nil { + t.Fatal(err) + } + + ft := time.Now() + err = ft.UnmarshalText([]byte(lines[0][0])) + if err != nil { + t.Fatal(err) + } + if ft.Sub(d) != 0 { + t.Fatalf("Dates doesn't match: %s and actual: %s", d, d) + } + + m, _ := d.MarshalText() + assertLine(t, []string{string(m)}, lines[0]) +} + +func Test_writeTo_NoHeaders(t *testing.T) { + b := bytes.Buffer{} + e := &encoder{out: &b} + blah := 2 + sptr := "*string" + s := []Sample{ + {Foo: "f", Bar: 1, Baz: "baz", Frop: 0.1, Blah: &blah, SPtr: &sptr}, + {Foo: "e", Bar: 3, Baz: "b", Frop: 6.0 / 13, Blah: nil, SPtr: nil}, + } + if err := writeTo(NewSafeCSVWriter(csv.NewWriter(e.out)), s, true); err != nil { + t.Fatal(err) + } + + lines, err := csv.NewReader(&b).ReadAll() + if err != nil { + t.Fatal(err) + } + if len(lines) != 2 { + t.Fatalf("expected 2 lines, got %d", len(lines)) + } + assertLine(t, []string{"f", "1", "baz", "0.1", "2", "*string", ""}, lines[0]) + assertLine(t, []string{"e", "3", "b", "0.46153846153846156", "", "", ""}, lines[1]) +} + +func Test_writeTo_multipleTags(t *testing.T) { + b := bytes.Buffer{} + e := &encoder{out: &b} + s := []MultiTagSample{ + {Foo: "abc", Bar: 123}, + {Foo: "def", Bar: 234}, + } + if err := writeTo(NewSafeCSVWriter(csv.NewWriter(e.out)), s, false); err != nil { + t.Fatal(err) + } + + lines, err := csv.NewReader(&b).ReadAll() + if err != nil { + t.Fatal(err) + } + if len(lines) != 3 { + t.Fatalf("expected 3 lines, got %d", len(lines)) + } + // the first tag for each field is the encoding CSV header + assertLine(t, []string{"Baz", "BAR"}, lines[0]) + assertLine(t, []string{"abc", "123"}, lines[1]) + assertLine(t, []string{"def", "234"}, lines[2]) +} + +func Test_writeTo_slice(t *testing.T) { + b := bytes.Buffer{} + e := &encoder{out: &b} + + type TestType struct { + Key string + Items []int + } + + s := []TestType{ + { + Key: "test1", + Items: []int{1, 2, 3}, + }, + { + Key: "test2", + Items: []int{4, 5, 6}, + }, + } + + if err := writeTo(NewSafeCSVWriter(csv.NewWriter(e.out)), s, false); err != nil { + t.Fatal(err) + } + + lines, err := csv.NewReader(&b).ReadAll() + if err != nil { + t.Fatal(err) + } + + if len(lines) != 3 { + t.Fatalf("expected 3 lines, got %d", len(lines)) + } + + assertLine(t, []string{"Key", "Items"}, lines[0]) + assertLine(t, []string{"test1", "[1,2,3]"}, lines[1]) + assertLine(t, []string{"test2", "[4,5,6]"}, lines[2]) +} + +func TestMarshalInterfaceSlice(t *testing.T) { + type sample struct { + Type string + Value interface{} + } + + csvContent, err := MarshalString(&[]sample{ + {Type: "numbers", Value: []int64{1, 2, 3}}, + }) + if err != nil { + t.Fatal(err) + } + + if expected := "Type,Value\nnumbers,\"[1,2,3]\"\n"; csvContent != expected { + t.Fatalf("expected interface slice to marshal as JSON.\nexpected: %q\ngot: %q", expected, csvContent) + } +} + +func Test_writeTo_slice_structs(t *testing.T) { + b := bytes.Buffer{} + e := &encoder{out: &b} + s := []SliceStructSample{ + { + Slice: []SliceStruct{ + {String: "s1", Float: 1.1}, + {String: "s2", Float: 2.2}, + {String: "nope", Float: 3.3}, + }, + Slice2: []SliceStruct{ + {String: "s1", Float: 1.1}, + {String: "s2", Float: 2.2}, + {String: "nope", Float: 3.3}, + }, + SimpleSlice: []int{1, 2, 3, 4, 5}, + Array: [2]SliceStruct{ + {String: "s3", Float: 3.3}, + {String: "s4", Float: 4.4}, + }, + }, + } + if err := writeTo(NewSafeCSVWriter(csv.NewWriter(e.out)), s, false); err != nil { + t.Fatal(err) + } + + lines, err := csv.NewReader(&b).ReadAll() + if err != nil { + t.Fatal(err) + } + if len(lines) != 2 { + t.Fatalf("expected 2 lines, got %d", len(lines)) + } + assertLine(t, []string{"s[0].s", "s[0].f", "s[1].s", "s[1].f", "sliceText", "ints[0]", "ints[1]", "ints[2]", "a[0].s", "a[0].f", "a[1].s", "a[1].f"}, lines[0]) + assertLine(t, []string{"s1", "1.1", "s2", "2.2", "[{\"String\":\"s1\",\"Float\":1.1},{\"String\":\"s2\",\"Float\":2.2},{\"String\":\"nope\",\"Float\":3.3}]", "1", "2", "3", "s3", "3.3", "s4", "4.4"}, lines[1]) +} + +func Test_writeTo_embed(t *testing.T) { + b := bytes.Buffer{} + e := &encoder{out: &b} + blah := 2 + sptr := "*string" + s := []EmbedSample{ + { + Qux: "aaa", + Sample: Sample{Foo: "f", Bar: 1, Baz: "baz", Frop: 0.2, Blah: &blah, SPtr: &sptr}, + Ignore: "shouldn't be marshalled", + Quux: "zzz", + Grault: math.Pi, + }, + } + if err := writeTo(NewSafeCSVWriter(csv.NewWriter(e.out)), s, false); err != nil { + t.Fatal(err) + } + + lines, err := csv.NewReader(&b).ReadAll() + if err != nil { + t.Fatal(err) + } + if len(lines) != 2 { + t.Fatalf("expected 2 lines, got %d", len(lines)) + } + assertLine(t, []string{"first", "foo", "BAR", "Baz", "Quux", "Blah", "SPtr", "Omit", "garply", "last"}, lines[0]) + assertLine(t, []string{"aaa", "f", "1", "baz", "0.2", "2", "*string", "", "3.141592653589793", "zzz"}, lines[1]) +} + +func Test_writeTo_embedptr(t *testing.T) { + b := bytes.Buffer{} + e := &encoder{out: &b} + blah := 2 + sptr := "*string" + s := []EmbedPtrSample{ + { + Qux: "aaa", + Sample: &Sample{Foo: "f", Bar: 1, Baz: "baz", Frop: 0.2, Blah: &blah, SPtr: &sptr}, + Ignore: "shouldn't be marshalled", + Quux: "zzz", + Grault: math.Pi, + }, + } + if err := writeTo(NewSafeCSVWriter(csv.NewWriter(e.out)), s, false); err != nil { + t.Fatal(err) + } + + lines, err := csv.NewReader(&b).ReadAll() + if err != nil { + t.Fatal(err) + } + if len(lines) != 2 { + t.Fatalf("expected 2 lines, got %d", len(lines)) + } + assertLine(t, []string{"first", "foo", "BAR", "Baz", "Quux", "Blah", "SPtr", "Omit", "garply", "last"}, lines[0]) + assertLine(t, []string{"aaa", "f", "1", "baz", "0.2", "2", "*string", "", "3.141592653589793", "zzz"}, lines[1]) +} + +func Test_writeTo_embedptr_nil(t *testing.T) { + b := bytes.Buffer{} + e := &encoder{out: &b} + s := []EmbedPtrSample{ + {}, + } + if err := writeTo(NewSafeCSVWriter(csv.NewWriter(e.out)), s, false); err != nil { + t.Fatal(err) + } + + lines, err := csv.NewReader(&b).ReadAll() + if err != nil { + t.Fatal(err) + } + if len(lines) != 2 { + t.Fatalf("expected 2 lines, got %d", len(lines)) + } + assertLine(t, []string{"first", "foo", "BAR", "Baz", "Quux", "Blah", "SPtr", "Omit", "garply", "last"}, lines[0]) + assertLine(t, []string{"", "", "", "", "", "", "", "", "0", ""}, lines[1]) +} + +func Test_writeTo_embedmarshal(t *testing.T) { + b := bytes.Buffer{} + e := &encoder{out: &b} + s := []EmbedMarshal{ + { + Foo: &MarshalSample{Dummy: "bar"}, + }, + } + if err := writeTo(NewSafeCSVWriter(csv.NewWriter(e.out)), s, false); err != nil { + t.Fatal(err) + } + + lines, err := csv.NewReader(&b).ReadAll() + if err != nil { + t.Fatal(err) + } + if len(lines) != 2 { + t.Fatalf("expected 2 lines, got %d", len(lines)) + } + assertLine(t, []string{"foo"}, lines[0]) + assertLine(t, []string{"bar"}, lines[1]) + +} + +func Test_writeTo_embedmarshalCSV(t *testing.T) { + + // First, create our test data + b := new(bytes.Buffer) + e := &encoder{out: b} + s := []*EmbedMarshalCSV{ + { + Symbol: "test", + Timestamp: &MarshalCSVSample{ + Seconds: 1656460798, + Nanos: 693201614, + }, + }, + } + + // Next, attempt to write our test data to a CSV format + if err := writeTo(NewSafeCSVWriter(csv.NewWriter(e.out)), s, false); err != nil { + t.Fatal(err) + } + + // Now, read in the data we just wrote + lines, err := csv.NewReader(b).ReadAll() + if err != nil { + t.Fatal(err) + } + + // Finally, verify the structure of the data + if len(lines) != 2 { + t.Fatalf("expected 2 lines, got %d", len(lines)) + } + + assertLine(t, []string{"symbol", "timestamp"}, lines[0]) + assertLine(t, []string{"test", "1656460798.693201614"}, lines[1]) +} + +func Test_writeTo_complex_embed(t *testing.T) { + b := bytes.Buffer{} + e := &encoder{out: &b} + sptr := "*string" + sfs := []SkipFieldSample{ + { + EmbedSample: EmbedSample{ + Qux: "aaa", + Sample: Sample{ + Foo: "bbb", + Bar: 111, + Baz: "ddd", + Frop: 1.2e22, + Blah: nil, + SPtr: &sptr, + }, + Ignore: "eee", + Grault: 0.1, + Quux: "fff", + }, + MoreIgnore: "ggg", + Corge: "hhh", + }, + } + if err := writeTo(NewSafeCSVWriter(csv.NewWriter(e.out)), sfs, false); err != nil { + t.Fatal(err) + } + lines, err := csv.NewReader(&b).ReadAll() + if err != nil { + t.Fatal(err) + } + if len(lines) != 2 { + t.Fatalf("expected 2 lines, got %d", len(lines)) + } + assertLine(t, []string{"first", "foo", "BAR", "Baz", "Quux", "Blah", "SPtr", "Omit", "garply", "last", "abc"}, lines[0]) + assertLine(t, []string{"aaa", "bbb", "111", "ddd", "12000000000000000000000", "", "*string", "", "0.1", "fff", "hhh"}, lines[1]) +} + +func Test_writeTo_complex_inner_struct_embed(t *testing.T) { + b := bytes.Buffer{} + e := &encoder{out: &b} + sfs := []Level0Struct{ + { + Level0Field: level1Struct{ + Level1Field: level2Struct{ + InnerStruct{ + BoolIgnoreField0: false, + BoolField1: false, + StringField2: "email1", + }, + }, + }, + }, + { + Level0Field: level1Struct{ + Level1Field: level2Struct{ + InnerStruct{ + BoolIgnoreField0: false, + BoolField1: true, + StringField2: "email2", + }, + }, + }, + }, + } + + if err := writeTo(NewSafeCSVWriter(csv.NewWriter(e.out)), sfs, true); err != nil { + t.Fatal(err) + } + lines, err := csv.NewReader(&b).ReadAll() + if err != nil { + t.Fatal(err) + } + assertLine(t, []string{"false", "email1"}, lines[0]) + assertLine(t, []string{"true", "email2"}, lines[1]) +} + +func Test_writeToChan(t *testing.T) { + b := bytes.Buffer{} + e := &encoder{out: &b} + c := make(chan interface{}) + sptr := "*string" + go func() { + for i := 0; i < 100; i++ { + v := Sample{Foo: "f", Bar: i, Baz: "baz" + strconv.Itoa(i), Frop: float64(i), Blah: nil, SPtr: &sptr} + c <- v + } + close(c) + }() + if err := MarshalChan(c, NewSafeCSVWriter(csv.NewWriter(e.out))); err != nil { + t.Fatal(err) + } + lines, err := csv.NewReader(&b).ReadAll() + if err != nil { + t.Fatal(err) + } + if len(lines) != 101 { + t.Fatalf("expected 100 lines, got %d", len(lines)) + } + for i, l := range lines { + if i == 0 { + assertLine(t, []string{"foo", "BAR", "Baz", "Quux", "Blah", "SPtr", "Omit"}, l) + continue + } + assertLine(t, []string{"f", strconv.Itoa(i - 1), "baz" + strconv.Itoa(i-1), strconv.FormatFloat(float64(i-1), 'f', -1, 64), "", "*string", ""}, l) + } +} + +func Test_MarshalChan_ClosedChannel(t *testing.T) { + b := bytes.Buffer{} + e := &encoder{out: &b} + c := make(chan interface{}) + close(c) + + if err := MarshalChan(c, NewSafeCSVWriter(csv.NewWriter(e.out))); !errors.Is(err, ErrChannelIsClosed) { + t.Fatal(err) + } +} + +// TestRenamedTypes tests for marshaling functions on redefined basic types. +func TestRenamedTypesMarshal(t *testing.T) { + samples := []RenamedSample{ + {RenamedFloatUnmarshaler: 1.4, RenamedFloatDefault: 1.5}, + {RenamedFloatUnmarshaler: 2.3, RenamedFloatDefault: 2.4}, + } + + SetCSVWriter(func(out io.Writer) *SafeCSVWriter { + csvout := NewSafeCSVWriter(csv.NewWriter(out)) + csvout.Comma = ';' + return csvout + }) + // Switch back to default for tests executed after this + defer SetCSVWriter(DefaultCSVWriter) + + csvContent, err := MarshalString(&samples) + if err != nil { + t.Fatal(err) + } + if csvContent != "foo;bar\n1,4;1.5\n2,3;2.4\n" { + t.Fatalf("Error marshaling floats with , as separator. Expected \nfoo;bar\n1,4;1.5\n2,3;2.4\ngot:\n%v", csvContent) + } + + // Test that errors raised by MarshalCSV are correctly reported + samples = []RenamedSample{ + {RenamedFloatUnmarshaler: 4.2, RenamedFloatDefault: 1.5}, + } + _, err = MarshalString(&samples) + if _, ok := err.(MarshalError); !ok { + t.Fatalf("Expected UnmarshalError, got %v", err) + } +} + +// TestColumnNameStartingWithPartial is a regression test for issue #274: a column name that +// happens to start with "partial" (e.g. "partial_delivery_number") must not be mistaken for the +// "partial" tag option. Previously the option was detected with strings.HasPrefix, so such a +// column name was swallowed and gocsv fell back to using the Go field name instead. +func TestColumnNameStartingWithPartial(t *testing.T) { + type sample struct { + PartialDeliveryNumber uint `csv:"partial_delivery_number"` + } + + // Marshal: the header must be the csv tag, not the Go field name. + csvContent, err := MarshalString(&[]sample{{PartialDeliveryNumber: 7}}) + if err != nil { + t.Fatal(err) + } + if expected := "partial_delivery_number\n7\n"; csvContent != expected { + t.Fatalf("expected header from csv tag.\nexpected: %q\ngot: %q", expected, csvContent) + } + + // Unmarshal: a CSV whose header is the csv tag must populate the field. + var out []sample + if err := Unmarshal(bytes.NewBufferString("partial_delivery_number\n42\n"), &out); err != nil { + t.Fatal(err) + } + if len(out) != 1 || out[0].PartialDeliveryNumber != 42 { + t.Fatalf("expected PartialDeliveryNumber=42, got %+v", out) + } +} + +// TestCustomTagSeparatorMarshal tests for custom tag separator in marshalling. +func TestCustomTagSeparatorMarshal(t *testing.T) { + samples := []RenamedSample{ + {RenamedFloatUnmarshaler: 1.4, RenamedFloatDefault: 1.5}, + {RenamedFloatUnmarshaler: 2.3, RenamedFloatDefault: 2.4}, + } + + TagSeparator = " | " + // Switch back to default TagSeparator after this + defer func() { + TagSeparator = "," + }() + + csvContent, err := MarshalString(&samples) + if err != nil { + t.Fatal(err) + } + if csvContent != "foo|bar\n1,4|1.5\n2,3|2.4\n" { + t.Fatalf("Error marshaling floats with , as separator. Expected \nfoo|bar\n1,4|1.5\n2,3|2.4\ngot:\n%v", csvContent) + } +} + +func (rf *RenamedFloat64Unmarshaler) MarshalCSV() (csv string, err error) { + if *rf == RenamedFloat64Unmarshaler(4.2) { + return "", MarshalError{"Test error: Invalid float 4.2"} + } + csv = strconv.FormatFloat(float64(*rf), 'f', 1, 64) + csv = strings.Replace(csv, ".", ",", -1) + return csv, nil +} + +type MarshalError struct { + msg string +} + +func (e MarshalError) Error() string { + return e.msg +} + +func Benchmark_MarshalCSVWithoutHeaders(b *testing.B) { + dst := NewSafeCSVWriter(csv.NewWriter(ioutil.Discard)) + for n := 0; n < b.N; n++ { + err := MarshalCSVWithoutHeaders([]Sample{{}}, dst) + if err != nil { + b.Fatal(err) + } + } +} + +func Test_writeTo_nested_struct(t *testing.T) { + b := bytes.Buffer{} + e := &encoder{out: &b} + s := []NestedSample{ + { + Inner1: InnerStruct{ + BoolIgnoreField0: false, + BoolField1: false, + StringField2: "email_one", + }, + Inner2: InnerStruct{ + BoolIgnoreField0: true, + BoolField1: true, + StringField2: "email_two", + }, + InnerIgnore: InnerStruct{ + BoolIgnoreField0: true, + BoolField1: false, + StringField2: "email_ignore", + }, + Inner3: NestedEmbedSample{InnerStruct{ + BoolIgnoreField0: true, + BoolField1: false, + StringField2: "email_three", + }}, + }, + } + if err := writeTo(NewSafeCSVWriter(csv.NewWriter(e.out)), s, false); err != nil { + t.Fatal(err) + } + + lines, err := csv.NewReader(&b).ReadAll() + if err != nil { + t.Fatal(err) + } + if len(lines) != 2 { + t.Fatalf("expected 2 lines, got %d", len(lines)) + } + assertLine(t, []string{"one.boolField1", "one.stringField2", "two.boolField1", "two.stringField2", "three.boolField1", "three.stringField2"}, lines[0]) + assertLine(t, []string{"false", "email_one", "true", "email_two", "false", "email_three"}, lines[1]) + t.Run("struct is a child element ", func(t *testing.T) { + type Nested struct { + UpdatedAt *time.Time `csv:"updated_at"` + } + type Row struct { + Name string `csv:"name"` + Data Nested `csv:"data"` + } + data := time.Date(2022, 12, 11, 13, 14, 15, 123456, time.UTC) + dat := []Row{{Name: "matthew", Data: Nested{UpdatedAt: &data}}} + got, err := MarshalString(dat) + if err != nil { + t.Fatal(err) + } + want := `name,data.updated_at +matthew,2022-12-11T13:14:15.000123456Z +` + if got != want { + t.Fatalf("Error marshaling floats with , as separator. Expected \n%s\ngot:\n%s", want, got) + } + }) + +} + +func Test_non_marshaling_nested_fields_are_prefixed(t *testing.T) { + b := bytes.Buffer{} + e := &encoder{out: &b} + time1 := time.Date(2021, 2, 19, 0, 0, 0, 0, time.UTC) + time2 := time.Date(2022, 2, 19, 0, 0, 0, 0, time.UTC) + time3 := time.Date(2023, 2, 19, 0, 0, 0, 0, time.UTC) + + s := []SameNameStruct{ + { + Inner2: &InnerStruct2{ + Bar: "bar1", + Inner3: InnerStruct3{Bar: "bar2", Foo: time1}, + }, + Inner3: InnerStruct3{Bar: "bar3", Foo: time2}, + Foo: time3, + }, + } + if err := writeTo(NewSafeCSVWriter(csv.NewWriter(e.out)), s, false); err != nil { + t.Fatal(err) + } + lines, err := csv.NewReader(&b).ReadAll() + if err != nil { + t.Fatal(err) + } + if len(lines) != 2 { + t.Fatalf("expected 2 lines, got %d", len(lines)) + } + // The headers should contain the struct path prefixes even if struct is responsible for its marshalling like time.Time + assertLine(t, []string{"Inner2.Bar", "Inner2.Inner3.Bar", "Inner2.Inner3.Foo", "Inner3.Bar", "Inner3.Foo", "Foo"}, lines[0]) + assertLine(t, []string{"bar1", "bar2", "2021-02-19T00:00:00Z", "bar3", "2022-02-19T00:00:00Z", "2023-02-19T00:00:00Z"}, lines[1]) +} + +func Test_writeTo_inline_nested_struct(t *testing.T) { + b := bytes.Buffer{} + e := &encoder{out: &b} + s := []InlineFooSample{ + {Bar: InlineBar{A: 1, B: 2}, X: 3}, + } + if err := writeTo(NewSafeCSVWriter(csv.NewWriter(e.out)), s, false); err != nil { + t.Fatal(err) + } + lines, err := csv.NewReader(&b).ReadAll() + if err != nil { + t.Fatal(err) + } + if len(lines) != 2 { + t.Fatalf("expected 2 lines, got %d", len(lines)) + } + // The `csv:"."` tag inlines Bar's fields without the "Bar." prefix. + assertLine(t, []string{"a", "b", "x"}, lines[0]) + assertLine(t, []string{"1", "2", "3"}, lines[1]) + + t.Run("pointer to struct", func(t *testing.T) { + got, err := MarshalString([]InlineFooPtrSample{{Bar: &InlineBar{A: 4, B: 5}, X: 6}}) + if err != nil { + t.Fatal(err) + } + want := "a,b,x\n4,5,6\n" + if got != want { + t.Fatalf("expected:\n%s\ngot:\n%s", want, got) + } + }) + + t.Run("inline struct nested under a prefixed field", func(t *testing.T) { + got, err := MarshalString([]InlineOuterSample{{Foo: InlineFooSample{Bar: InlineBar{A: 1, B: 2}, X: 3}, Y: 4}}) + if err != nil { + t.Fatal(err) + } + // Foo is prefixed with "foo", and Bar's "." tag drops only its own name, + // so Bar's fields keep the inherited "foo." prefix. + want := "foo.a,foo.b,foo.x,y\n1,2,3,4\n" + if got != want { + t.Fatalf("expected:\n%s\ngot:\n%s", want, got) + } + }) +}