Compare commits

..
10 Commits
Author SHA1 Message Date
dualfroz b4654a48b0 Describe this repository 2026-10-22 21:03:58 +02:00
dualfroz f87ec7522c Make SetHeaderNormalizer safe to call while decoding
SetHeaderNormalizer reassigned the normalizer and copied a fresh sync.Map
over the struct info cache while other goroutines were using it. Keep both
in one value swapped atomically instead. Fixes #237.
2026-10-02 16:20:31 +02:00
upstream 391e8f0876 Upstream gocarina/gocsv state of the touched files before ddaa038afb 2026-10-02 16:20:31 +02:00
dualfroz 5f7829feee Fix date layout in README and full example 2026-10-02 15:53:20 +02:00
upstream a5ece792f8 Upstream gocarina/gocsv state of the touched files before c31659ffab 2026-10-02 15:53:20 +02:00
dualfroz 90a1c52508 Add MarshalBytesWithoutHeaders 2026-10-02 15:52:52 +02:00
upstream 4d32abefad Upstream gocarina/gocsv state of the touched files before 8fabd92666 2026-10-02 15:52:52 +02:00
dualfroz 26aafe16a8 fix: parse integer fields as base 10
toInt and toUint passed base 0 to strconv, which infers the base from the
literal the way the Go compiler does. A zero-padded CSV field was then
read as octal: 08 and 09 failed with invalid syntax and 010 silently
parsed as 8.

Pass base 10 at both call sites. Go-style prefixed literals such as 0x1f
are no longer accepted; they were undocumented and untested, and cannot
be kept without keeping the octal reading of 010 as well.

Closes #182
2026-09-04 07:41:12 +02:00
upstream 0ba11f2f3b Upstream gocarina/gocsv state of the touched files before 7c8185d25e 2026-09-04 07:41:12 +02:00
dualfroz 9bc91ef163 fix: parse integer fields as base 10
toInt and toUint passed base 0 to strconv, which infers the base from the
literal the way the Go compiler does. A zero-padded CSV field was then
read as octal: 08 and 09 failed with invalid syntax and 010 silently
parsed as 8.

Pass base 10 at both call sites. Go-style prefixed literals such as 0x1f
are no longer accepted; they were undocumented and untested, and cannot
be kept without keeping the octal reading of 010 as well.

Closes #182
2026-09-04 22:08:59 +02:00
10 changed files with 1935 additions and 2 deletions

No files matched your search

+13
View File
@@ -0,0 +1,13 @@
# Contributions to gocarina/gocsv
The GoCSV package aims to provide easy CSV serialization and deserialization to the golang programming language
This repository holds only my commits to the upstream project, with their original author, date and diff.
Each of my commits is preceded by a snapshot of the upstream files it touched, so every diff shows exactly what I changed.
## Merged pull requests
- [Add MarshalBytesWithoutHeaders](https://github.com/gocarina/gocsv/pull/305) (2026-10-12)
- [Fix date layout in README and full example](https://github.com/gocarina/gocsv/pull/304) (2026-10-12)
- [Make SetHeaderNormalizer safe to call while decoding](https://github.com/gocarina/gocsv/pull/303) (2026-10-12)
- [Parse integer fields as base 10 (fixes #182)](https://github.com/gocarina/gocsv/pull/293) (2026-09-05)
+586
View File
@@ -0,0 +1,586 @@
// Copyright 2014 Jonathan Picques. All rights reserved.
// Use of this source code is governed by a MIT license
// The license can be found in the LICENSE file.
// The GoCSV package aims to provide easy CSV serialization and deserialization to the golang programming language
package gocsv
import (
"bytes"
"encoding/csv"
"fmt"
"io"
"mime/multipart"
"os"
"reflect"
"strings"
"sync"
"sync/atomic"
)
// FailIfUnmatchedStructTags indicates whether it is considered an error when there is an unmatched
// struct tag.
var FailIfUnmatchedStructTags = false
// FailIfDoubleHeaderNames indicates whether it is considered an error when a header name is repeated
// in the csv header.
var FailIfDoubleHeaderNames = false
// ShouldAlignDuplicateHeadersWithStructFieldOrder indicates whether we should align duplicate CSV
// headers per their alignment in the struct definition.
var ShouldAlignDuplicateHeadersWithStructFieldOrder = false
// TagName defines key in the struct field's tag to scan
var TagName = "csv"
// TagSeparator defines seperator string for multiple csv tags in struct fields
var TagSeparator = ","
// FieldSeperator defines how to combine parent struct with child struct
var FieldsCombiner = "."
// Normalizer is a function that takes and returns a string. It is applied to
// struct and header field values before they are compared. It can be used to alter
// names for comparison. For instance, you could allow case insensitive matching
// or convert '-' to '_'.
type Normalizer func(string) string
type ErrorHandler func(*csv.ParseError) bool
// headerNormalization is the Normalizer in use together with the struct infos
// built with it. SetHeaderNormalizer replaces both at once, so it is safe to
// call while other goroutines are reading or writing CSV.
type headerNormalization struct {
normalize Normalizer
structInfos sync.Map
}
// currentHeaderNormalization holds a *headerNormalization, initially with a nop Normalizer.
var currentHeaderNormalization atomic.Value
func init() {
currentHeaderNormalization.Store(&headerNormalization{normalize: DefaultNameNormalizer()})
}
func loadHeaderNormalization() *headerNormalization {
return currentHeaderNormalization.Load().(*headerNormalization)
}
func normalizeName(name string) string {
return loadHeaderNormalization().normalize(name)
}
// DefaultNameNormalizer is a nop Normalizer.
func DefaultNameNormalizer() Normalizer { return func(s string) string { return s } }
// SetHeaderNormalizer sets the normalizer used to normalize struct and header field names.
func SetHeaderNormalizer(f Normalizer) {
// The struct infos are rebuilt from scratch, as they hold normalized names.
currentHeaderNormalization.Store(&headerNormalization{normalize: f})
}
// --------------------------------------------------------------------------
// CSVWriter used to format CSV
var selfCSVWriter = DefaultCSVWriter
// DefaultCSVWriter is the default SafeCSVWriter used to format CSV (cf. csv.NewWriter)
func DefaultCSVWriter(out io.Writer) *SafeCSVWriter {
writer := NewSafeCSVWriter(csv.NewWriter(out))
// As only one rune can be defined as a CSV separator, we are going to trim
// the custom tag separator and use the first rune.
if runes := []rune(strings.TrimSpace(TagSeparator)); len(runes) > 0 {
writer.Comma = runes[0]
}
return writer
}
// SetCSVWriter sets the SafeCSVWriter used to format CSV.
func SetCSVWriter(csvWriter func(io.Writer) *SafeCSVWriter) {
selfCSVWriter = csvWriter
}
func getCSVWriter(out io.Writer) *SafeCSVWriter {
return selfCSVWriter(out)
}
// --------------------------------------------------------------------------
// CSVReader used to parse CSV
var selfCSVReader = DefaultCSVReader
// DefaultCSVReader is the default CSV reader used to parse CSV (cf. csv.NewReader)
func DefaultCSVReader(in io.Reader) CSVReader {
return csv.NewReader(in)
}
// LazyCSVReader returns a lazy CSV reader, with LazyQuotes and TrimLeadingSpace.
func LazyCSVReader(in io.Reader) CSVReader {
csvReader := csv.NewReader(in)
csvReader.LazyQuotes = true
csvReader.TrimLeadingSpace = true
return csvReader
}
// SetCSVReader sets the CSV reader used to parse CSV.
func SetCSVReader(csvReader func(io.Reader) CSVReader) {
selfCSVReader = csvReader
}
func getCSVReader(in io.Reader) CSVReader {
return selfCSVReader(in)
}
// --------------------------------------------------------------------------
// Marshal functions
// MarshalFile saves the interface as CSV in the file.
func MarshalFile(in interface{}, file *os.File) (err error) {
return Marshal(in, file)
}
// MarshalString returns the CSV string from the interface.
func MarshalString(in interface{}) (out string, err error) {
bufferString := bytes.NewBufferString(out)
if err := Marshal(in, bufferString); err != nil {
return "", err
}
return bufferString.String(), nil
}
// MarshalStringWithoutHeaders returns the CSV string from the interface.
func MarshalStringWithoutHeaders(in interface{}) (out string, err error) {
bufferString := bytes.NewBufferString(out)
if err := MarshalWithoutHeaders(in, bufferString); err != nil {
return "", err
}
return bufferString.String(), nil
}
// MarshalBytes returns the CSV bytes from the interface.
func MarshalBytes(in interface{}) (out []byte, err error) {
bufferString := bytes.NewBuffer(out)
if err := Marshal(in, bufferString); err != nil {
return nil, err
}
return bufferString.Bytes(), nil
}
// Marshal returns the CSV in writer from the interface.
func Marshal(in interface{}, out io.Writer) (err error) {
writer := getCSVWriter(out)
return writeTo(writer, in, false)
}
// MarshalWithoutHeaders returns the CSV in writer from the interface.
func MarshalWithoutHeaders(in interface{}, out io.Writer) (err error) {
writer := getCSVWriter(out)
return writeTo(writer, in, true)
}
// MarshalChan returns the CSV read from the channel.
func MarshalChan(c <-chan interface{}, out CSVWriter) error {
return writeFromChan(out, c, false)
}
// MarshalChanWithoutHeaders returns the CSV read from the channel.
func MarshalChanWithoutHeaders(c <-chan interface{}, out CSVWriter) error {
return writeFromChan(out, c, true)
}
// MarshalCSV returns the CSV in writer from the interface.
func MarshalCSV(in interface{}, out CSVWriter) (err error) {
return writeTo(out, in, false)
}
// MarshalCSVWithoutHeaders returns the CSV in writer from the interface.
func MarshalCSVWithoutHeaders(in interface{}, out CSVWriter) (err error) {
return writeTo(out, in, true)
}
// --------------------------------------------------------------------------
// Unmarshal functions
// UnmarshalFile parses the CSV from the file in the interface.
func UnmarshalFile(in *os.File, out interface{}) error {
return Unmarshal(in, out)
}
// UnmarshalMultipartFile parses the CSV from the multipart file in the interface.
func UnmarshalMultipartFile(in *multipart.File, out interface{}) error {
return Unmarshal(convertTo(in), out)
}
// UnmarshalFileWithErrorHandler parses the CSV from the file in the interface.
func UnmarshalFileWithErrorHandler(in *os.File, errHandler ErrorHandler, out interface{}) error {
return UnmarshalWithErrorHandler(in, errHandler, out)
}
// UnmarshalString parses the CSV from the string in the interface.
func UnmarshalString(in string, out interface{}) error {
return Unmarshal(strings.NewReader(in), out)
}
// UnmarshalBytes parses the CSV from the bytes in the interface.
func UnmarshalBytes(in []byte, out interface{}) error {
return Unmarshal(bytes.NewReader(in), out)
}
// Unmarshal parses the CSV from the reader in the interface.
func Unmarshal(in io.Reader, out interface{}) error {
return readTo(newSimpleDecoderFromReader(in), out)
}
// Unmarshal parses the CSV from the reader in the interface.
func UnmarshalWithErrorHandler(in io.Reader, errHandle ErrorHandler, out interface{}) error {
return readToWithErrorHandler(newSimpleDecoderFromReader(in), errHandle, out)
}
// UnmarshalWithoutHeaders parses the CSV from the reader in the interface.
func UnmarshalWithoutHeaders(in io.Reader, out interface{}) error {
return readToWithoutHeaders(newSimpleDecoderFromReader(in), out)
}
// UnmarshalCSVWithoutHeaders parses a headerless CSV with passed in CSV reader
func UnmarshalCSVWithoutHeaders(in CSVReader, out interface{}) error {
return readToWithoutHeaders(csvDecoder{in}, out)
}
// UnmarshalDecoder parses the CSV from the decoder in the interface
func UnmarshalDecoder(in Decoder, out interface{}) error {
return readTo(in, out)
}
// UnmarshalCSV parses the CSV from the reader in the interface.
func UnmarshalCSV(in CSVReader, out interface{}) error {
return readTo(csvDecoder{in}, out)
}
// UnmarshalCSVToMap parses a CSV of 2 columns into a map.
func UnmarshalCSVToMap(in CSVReader, out interface{}) error {
decoder := NewSimpleDecoderFromCSVReader(in)
header, err := decoder.GetCSVRow()
if err != nil {
return err
}
if len(header) != 2 {
return fmt.Errorf("maps can only be created for csv of two columns")
}
outValue, outType := getConcreteReflectValueAndType(out)
if outType.Kind() != reflect.Map {
return fmt.Errorf("cannot use " + outType.String() + ", only map supported")
}
keyType := outType.Key()
valueType := outType.Elem()
outValue.Set(reflect.MakeMap(outType))
for {
key := reflect.New(keyType)
value := reflect.New(valueType)
line, err := decoder.GetCSVRow()
if err == io.EOF {
break
} else if err != nil {
return err
}
if err := setField(key, line[0], false); err != nil {
return err
}
if err := setField(value, line[1], false); err != nil {
return err
}
outValue.SetMapIndex(key.Elem(), value.Elem())
}
return nil
}
// UnmarshalToChan parses the CSV from the reader and send each value in the chan c.
// The channel must have a concrete type.
func UnmarshalToChan(in io.Reader, c interface{}) error {
if c == nil {
return fmt.Errorf("goscv: channel is %v", c)
}
return readEach(newSimpleDecoderFromReader(in), nil, c)
}
// UnmarshalToChanWithErrorHandler parses the CSV from the reader in the interface.
func UnmarshalToChanWithErrorHandler(in io.Reader, errorHandler ErrorHandler, c interface{}) error {
if c == nil {
return fmt.Errorf("goscv: channel is %v", c)
}
return readEach(newSimpleDecoderFromReader(in), errorHandler, c)
}
// UnmarshalToChanWithoutHeaders parses the CSV from the reader and send each value in the chan c.
// The channel must have a concrete type.
func UnmarshalToChanWithoutHeaders(in io.Reader, c interface{}) error {
if c == nil {
return fmt.Errorf("goscv: channel is %v", c)
}
return readEachWithoutHeaders(newSimpleDecoderFromReader(in), c)
}
// UnmarshalDecoderToChan parses the CSV from the decoder and send each value in the chan c.
// The channel must have a concrete type.
func UnmarshalDecoderToChan(in SimpleDecoder, c interface{}) error {
if c == nil {
return fmt.Errorf("goscv: channel is %v", c)
}
return readEach(in, nil, c)
}
// UnmarshalStringToChan parses the CSV from the string and send each value in the chan c.
// The channel must have a concrete type.
func UnmarshalStringToChan(in string, c interface{}) error {
return UnmarshalToChan(strings.NewReader(in), c)
}
// UnmarshalBytesToChan parses the CSV from the bytes and send each value in the chan c.
// The channel must have a concrete type.
func UnmarshalBytesToChan(in []byte, c interface{}) error {
return UnmarshalToChan(bytes.NewReader(in), c)
}
// UnmarshalToCallback parses the CSV from the reader and send each value to the given func f.
// The func must look like func(Struct). If its last return value is an error,
// the parser is drained without further callback calls before returning that error.
func UnmarshalToCallback(in io.Reader, f interface{}) error {
valueFunc := reflect.ValueOf(f)
t := reflect.TypeOf(f)
if t.NumIn() != 1 {
return fmt.Errorf("the given function must have exactly one parameter")
}
cerr := make(chan error)
c := reflect.MakeChan(reflect.ChanOf(reflect.BothDir, t.In(0)), 0)
go func() {
cerr <- UnmarshalToChan(in, c.Interface())
}()
var callbackErr error
for {
select {
case err := <-cerr:
if callbackErr != nil {
return callbackErr
}
return err
default:
}
v, notClosed := c.Recv()
if !notClosed || v.Interface() == nil {
break
}
if callbackErr != nil {
continue // Drain the producer without invoking the callback again.
}
callResults := valueFunc.Call([]reflect.Value{v})
// Preserve a callback error while allowing the producer to finish.
if len(callResults) > 0 {
if err, ok := callResults[len(callResults)-1].Interface().(error); ok {
callbackErr = err
}
}
}
err := <-cerr
if callbackErr != nil {
return callbackErr
}
return err
}
// UnmarshalDecoderToCallback parses the CSV from the decoder and send each value to the given func f.
// The func must look like func(Struct).
func UnmarshalDecoderToCallback(in SimpleDecoder, f interface{}) error {
valueFunc := reflect.ValueOf(f)
t := reflect.TypeOf(f)
if t.NumIn() != 1 {
return fmt.Errorf("the given function must have exactly one parameter")
}
cerr := make(chan error)
c := reflect.MakeChan(reflect.ChanOf(reflect.BothDir, t.In(0)), 0)
go func() {
cerr <- UnmarshalDecoderToChan(in, c.Interface())
}()
for {
select {
case err := <-cerr:
return err
default:
}
v, notClosed := c.Recv()
if !notClosed || v.Interface() == nil {
break
}
valueFunc.Call([]reflect.Value{v})
}
return <-cerr
}
// UnmarshalBytesToCallback parses the CSV from the bytes and send each value to the given func f.
// The func must look like func(Struct).
func UnmarshalBytesToCallback(in []byte, f interface{}) error {
return UnmarshalToCallback(bytes.NewReader(in), f)
}
// UnmarshalStringToCallback parses the CSV from the string and send each value to the given func f.
// The func must look like func(Struct).
func UnmarshalStringToCallback(in string, c interface{}) (err error) {
return UnmarshalToCallback(strings.NewReader(in), c)
}
// UnmarshalToCallbackWithError parses the CSV from the reader and
// send each value to the given func f.
//
// If func returns error, it will stop processing, drain the
// parser and propagate the error to caller.
//
// The func must look like func(Struct) error.
func UnmarshalToCallbackWithError(in io.Reader, f interface{}) error {
valueFunc := reflect.ValueOf(f)
t := reflect.TypeOf(f)
if t.NumIn() != 1 {
return fmt.Errorf("the given function must have exactly one parameter")
}
if t.NumOut() != 1 {
return fmt.Errorf("the given function must have exactly one return value")
}
if !isErrorType(t.Out(0)) {
return fmt.Errorf("the given function must only return error")
}
cerr := make(chan error)
c := reflect.MakeChan(reflect.ChanOf(reflect.BothDir, t.In(0)), 0)
go func() {
cerr <- UnmarshalToChan(in, c.Interface())
}()
var fErr error
for {
select {
case err := <-cerr:
if err != nil {
return err
}
return fErr
default:
}
v, notClosed := c.Recv()
if !notClosed || v.Interface() == nil {
if err := <-cerr; err != nil {
fErr = err
}
break
}
// callback f has already returned an error, stop processing but keep draining the chan c
if fErr != nil {
continue
}
results := valueFunc.Call([]reflect.Value{v})
// If the callback f returns an error, stores it and returns it in future.
errValue := results[0].Interface()
if errValue != nil {
fErr = errValue.(error)
}
}
return fErr
}
// UnmarshalBytesToCallbackWithError parses the CSV from the bytes and
// send each value to the given func f.
//
// If func returns error, it will stop processing, drain the
// parser and propagate the error to caller.
//
// The func must look like func(Struct) error.
func UnmarshalBytesToCallbackWithError(in []byte, f interface{}) error {
return UnmarshalToCallbackWithError(bytes.NewReader(in), f)
}
// UnmarshalStringToCallbackWithError parses the CSV from the string and
// send each value to the given func f.
//
// If func returns error, it will stop processing, drain the
// parser and propagate the error to caller.
//
// The func must look like func(Struct) error.
func UnmarshalStringToCallbackWithError(in string, c interface{}) (err error) {
return UnmarshalToCallbackWithError(strings.NewReader(in), c)
}
// CSVToMap creates a simple map from a CSV of 2 columns.
func CSVToMap(in io.Reader) (map[string]string, error) {
decoder := newSimpleDecoderFromReader(in)
header, err := decoder.GetCSVRow()
if err != nil {
return nil, err
}
if len(header) != 2 {
return nil, fmt.Errorf("maps can only be created for csv of two columns")
}
m := make(map[string]string)
for {
line, err := decoder.GetCSVRow()
if err == io.EOF {
break
} else if err != nil {
return nil, err
}
m[line[0]] = line[1]
}
return m, nil
}
// CSVToMaps takes a reader and returns an array of dictionaries, using the header row as the keys
func CSVToMaps(reader io.Reader) ([]map[string]string, error) {
r := getCSVReader(reader)
rows := []map[string]string{}
var header []string
for {
record, err := r.Read()
if err == io.EOF {
break
}
if err != nil {
return nil, err
}
if header == nil {
header = record
} else {
dict := map[string]string{}
for i := range header {
dict[header[i]] = record[i]
}
rows = append(rows, dict)
}
}
return rows, nil
}
// CSVToChanMaps parses the CSV from the reader and send a dictionary in the chan c, using the header row as the keys.
func CSVToChanMaps(reader io.Reader, c chan<- map[string]string) error {
r := csv.NewReader(reader)
var header []string
for {
record, err := r.Read()
if err == io.EOF {
break
}
if err != nil {
return err
}
if header == nil {
header = record
} else {
dict := map[string]string{}
for i := range header {
dict[header[i]] = record[i]
}
c <- dict
}
}
return nil
}
+66
View File
@@ -0,0 +1,66 @@
package gocsv
import (
"errors"
"strings"
"sync"
"testing"
)
func TestUnmarshalToCallback_ReaderError(t *testing.T) {
type Dummy struct{}
var reader = &errorReader{}
err := UnmarshalToCallback(reader, func(Dummy) {})
if !errors.Is(err, readerErr) {
t.Error("UnmarshalToCallback should return first reader error")
}
err = UnmarshalDecoderToCallback(newSimpleDecoderFromReader(reader), func(Dummy) {})
if !errors.Is(err, readerErr) {
t.Error("UnmarshalDecoderToCallback should return first reader error")
}
err = UnmarshalToCallbackWithError(reader, func(Dummy) error { return nil })
if !errors.Is(err, readerErr) {
t.Error("UnmarshalToCallbackWithError should return first reader error")
}
}
func TestSetHeaderNormalizerWhileUnmarshaling(t *testing.T) {
defer SetHeaderNormalizer(DefaultNameNormalizer())
type row struct {
ID string `csv:"id"`
Name string `csv:"name"`
}
var wg sync.WaitGroup
for i := 0; i < 50; i++ {
wg.Add(2)
go func() {
defer wg.Done()
SetHeaderNormalizer(strings.ToLower)
}()
go func() {
defer wg.Done()
var rows []row
if err := UnmarshalString("id,name\n1,foo\n", &rows); err != nil {
t.Error(err)
return
}
if len(rows) != 1 || rows[0].ID != "1" || rows[0].Name != "foo" {
t.Errorf("unexpected rows: %+v", rows)
}
}()
}
wg.Wait()
}
type errorReader struct{}
func (e *errorReader) Read([]byte) (n int, err error) {
return 0, readerErr
}
var readerErr = errors.New("reader error")
+30
View File
@@ -1359,3 +1359,33 @@ func Test_readTo_inline_nested_struct(t *testing.T) {
t.Fatalf("expected \n sample: %v\n got: %v", expected, samples) t.Fatalf("expected \n sample: %v\n got: %v", expected, samples)
} }
} }
func TestUnmarshalZeroPaddedNumbers(t *testing.T) {
type row struct {
Name string `csv:"name"`
Code int `csv:"code"`
Count uint `csv:"count"`
}
in := "name,code,count\n" +
"a,08,08\n" +
"b,09,09\n" +
"c,010,010\n" +
"d,007,007\n"
var got []row
if err := UnmarshalString(in, &got); err != nil {
t.Fatalf("UnmarshalString failed on zero-padded numbers: %s", err)
}
want := []row{
{Name: "a", Code: 8, Count: 8},
{Name: "b", Code: 9, Count: 9},
{Name: "c", Code: 10, Count: 10},
{Name: "d", Code: 7, Count: 7},
}
if !reflect.DeepEqual(got, want) {
t.Errorf("UnmarshalString returned %+v, want %+v", got, want)
}
}
+743
View File
@@ -0,0 +1,743 @@
package gocsv
import (
"bytes"
"encoding/csv"
"errors"
"io"
"io/ioutil"
"math"
"strconv"
"strings"
"testing"
"time"
)
func assertLine(t *testing.T, expected, actual []string) {
if len(expected) != len(actual) {
t.Fatalf("line length mismatch between expected: %d and actual: %d\nExpected:\n%v\nActual:\n%v\n", len(expected), len(actual), expected, actual)
}
for i := range expected {
if expected[i] != actual[i] {
t.Fatalf("mismatch on field %d at line `%s`: %s != %s", i, expected, expected[i], actual[i])
}
}
}
func Test_writeTo(t *testing.T) {
b := bytes.Buffer{}
e := &encoder{out: &b}
blah := 2
sptr := "*string"
s := []Sample{
{Foo: "f", Bar: 1, Baz: "baz", Frop: 0.1, Blah: &blah, SPtr: &sptr},
{Foo: "e", Bar: 3, Baz: "b", Frop: 6.0 / 13, Blah: nil, SPtr: nil},
}
if err := writeTo(NewSafeCSVWriter(csv.NewWriter(e.out)), s, false); err != nil {
t.Fatal(err)
}
lines, err := csv.NewReader(&b).ReadAll()
if err != nil {
t.Fatal(err)
}
if len(lines) != 3 {
t.Fatalf("expected 3 lines, got %d", len(lines))
}
assertLine(t, []string{"foo", "BAR", "Baz", "Quux", "Blah", "SPtr", "Omit"}, lines[0])
assertLine(t, []string{"f", "1", "baz", "0.1", "2", "*string", ""}, lines[1])
assertLine(t, []string{"e", "3", "b", "0.46153846153846156", "", "", ""}, lines[2])
}
func Test_writeTo_Time(t *testing.T) {
b := bytes.Buffer{}
e := &encoder{out: &b}
d := time.Unix(60, 0)
s := []DateTime{
{Foo: d},
}
if err := writeTo(NewSafeCSVWriter(csv.NewWriter(e.out)), s, true); err != nil {
t.Fatal(err)
}
lines, err := csv.NewReader(&b).ReadAll()
if err != nil {
t.Fatal(err)
}
ft := time.Now()
err = ft.UnmarshalText([]byte(lines[0][0]))
if err != nil {
t.Fatal(err)
}
if ft.Sub(d) != 0 {
t.Fatalf("Dates doesn't match: %s and actual: %s", d, d)
}
m, _ := d.MarshalText()
assertLine(t, []string{string(m)}, lines[0])
}
func Test_writeTo_NoHeaders(t *testing.T) {
b := bytes.Buffer{}
e := &encoder{out: &b}
blah := 2
sptr := "*string"
s := []Sample{
{Foo: "f", Bar: 1, Baz: "baz", Frop: 0.1, Blah: &blah, SPtr: &sptr},
{Foo: "e", Bar: 3, Baz: "b", Frop: 6.0 / 13, Blah: nil, SPtr: nil},
}
if err := writeTo(NewSafeCSVWriter(csv.NewWriter(e.out)), s, true); err != nil {
t.Fatal(err)
}
lines, err := csv.NewReader(&b).ReadAll()
if err != nil {
t.Fatal(err)
}
if len(lines) != 2 {
t.Fatalf("expected 2 lines, got %d", len(lines))
}
assertLine(t, []string{"f", "1", "baz", "0.1", "2", "*string", ""}, lines[0])
assertLine(t, []string{"e", "3", "b", "0.46153846153846156", "", "", ""}, lines[1])
}
func TestMarshalBytesWithoutHeaders(t *testing.T) {
s := []Sample{
{Foo: "f", Bar: 1, Baz: "baz", Frop: 0.1},
{Foo: "e", Bar: 3, Baz: "b", Frop: 0.5},
}
csvContent, err := MarshalBytesWithoutHeaders(&s)
if err != nil {
t.Fatal(err)
}
if expected := "f,1,baz,0.1,,,\ne,3,b,0.5,,,\n"; string(csvContent) != expected {
t.Fatalf("expected: %q\ngot: %q", expected, csvContent)
}
}
func Test_writeTo_multipleTags(t *testing.T) {
b := bytes.Buffer{}
e := &encoder{out: &b}
s := []MultiTagSample{
{Foo: "abc", Bar: 123},
{Foo: "def", Bar: 234},
}
if err := writeTo(NewSafeCSVWriter(csv.NewWriter(e.out)), s, false); err != nil {
t.Fatal(err)
}
lines, err := csv.NewReader(&b).ReadAll()
if err != nil {
t.Fatal(err)
}
if len(lines) != 3 {
t.Fatalf("expected 3 lines, got %d", len(lines))
}
// the first tag for each field is the encoding CSV header
assertLine(t, []string{"Baz", "BAR"}, lines[0])
assertLine(t, []string{"abc", "123"}, lines[1])
assertLine(t, []string{"def", "234"}, lines[2])
}
func Test_writeTo_slice(t *testing.T) {
b := bytes.Buffer{}
e := &encoder{out: &b}
type TestType struct {
Key string
Items []int
}
s := []TestType{
{
Key: "test1",
Items: []int{1, 2, 3},
},
{
Key: "test2",
Items: []int{4, 5, 6},
},
}
if err := writeTo(NewSafeCSVWriter(csv.NewWriter(e.out)), s, false); err != nil {
t.Fatal(err)
}
lines, err := csv.NewReader(&b).ReadAll()
if err != nil {
t.Fatal(err)
}
if len(lines) != 3 {
t.Fatalf("expected 3 lines, got %d", len(lines))
}
assertLine(t, []string{"Key", "Items"}, lines[0])
assertLine(t, []string{"test1", "[1,2,3]"}, lines[1])
assertLine(t, []string{"test2", "[4,5,6]"}, lines[2])
}
func TestMarshalInterfaceSlice(t *testing.T) {
type sample struct {
Type string
Value interface{}
}
csvContent, err := MarshalString(&[]sample{
{Type: "numbers", Value: []int64{1, 2, 3}},
})
if err != nil {
t.Fatal(err)
}
if expected := "Type,Value\nnumbers,\"[1,2,3]\"\n"; csvContent != expected {
t.Fatalf("expected interface slice to marshal as JSON.\nexpected: %q\ngot: %q", expected, csvContent)
}
}
func Test_writeTo_slice_structs(t *testing.T) {
b := bytes.Buffer{}
e := &encoder{out: &b}
s := []SliceStructSample{
{
Slice: []SliceStruct{
{String: "s1", Float: 1.1},
{String: "s2", Float: 2.2},
{String: "nope", Float: 3.3},
},
Slice2: []SliceStruct{
{String: "s1", Float: 1.1},
{String: "s2", Float: 2.2},
{String: "nope", Float: 3.3},
},
SimpleSlice: []int{1, 2, 3, 4, 5},
Array: [2]SliceStruct{
{String: "s3", Float: 3.3},
{String: "s4", Float: 4.4},
},
},
}
if err := writeTo(NewSafeCSVWriter(csv.NewWriter(e.out)), s, false); err != nil {
t.Fatal(err)
}
lines, err := csv.NewReader(&b).ReadAll()
if err != nil {
t.Fatal(err)
}
if len(lines) != 2 {
t.Fatalf("expected 2 lines, got %d", len(lines))
}
assertLine(t, []string{"s[0].s", "s[0].f", "s[1].s", "s[1].f", "sliceText", "ints[0]", "ints[1]", "ints[2]", "a[0].s", "a[0].f", "a[1].s", "a[1].f"}, lines[0])
assertLine(t, []string{"s1", "1.1", "s2", "2.2", "[{\"String\":\"s1\",\"Float\":1.1},{\"String\":\"s2\",\"Float\":2.2},{\"String\":\"nope\",\"Float\":3.3}]", "1", "2", "3", "s3", "3.3", "s4", "4.4"}, lines[1])
}
func Test_writeTo_embed(t *testing.T) {
b := bytes.Buffer{}
e := &encoder{out: &b}
blah := 2
sptr := "*string"
s := []EmbedSample{
{
Qux: "aaa",
Sample: Sample{Foo: "f", Bar: 1, Baz: "baz", Frop: 0.2, Blah: &blah, SPtr: &sptr},
Ignore: "shouldn't be marshalled",
Quux: "zzz",
Grault: math.Pi,
},
}
if err := writeTo(NewSafeCSVWriter(csv.NewWriter(e.out)), s, false); err != nil {
t.Fatal(err)
}
lines, err := csv.NewReader(&b).ReadAll()
if err != nil {
t.Fatal(err)
}
if len(lines) != 2 {
t.Fatalf("expected 2 lines, got %d", len(lines))
}
assertLine(t, []string{"first", "foo", "BAR", "Baz", "Quux", "Blah", "SPtr", "Omit", "garply", "last"}, lines[0])
assertLine(t, []string{"aaa", "f", "1", "baz", "0.2", "2", "*string", "", "3.141592653589793", "zzz"}, lines[1])
}
func Test_writeTo_embedptr(t *testing.T) {
b := bytes.Buffer{}
e := &encoder{out: &b}
blah := 2
sptr := "*string"
s := []EmbedPtrSample{
{
Qux: "aaa",
Sample: &Sample{Foo: "f", Bar: 1, Baz: "baz", Frop: 0.2, Blah: &blah, SPtr: &sptr},
Ignore: "shouldn't be marshalled",
Quux: "zzz",
Grault: math.Pi,
},
}
if err := writeTo(NewSafeCSVWriter(csv.NewWriter(e.out)), s, false); err != nil {
t.Fatal(err)
}
lines, err := csv.NewReader(&b).ReadAll()
if err != nil {
t.Fatal(err)
}
if len(lines) != 2 {
t.Fatalf("expected 2 lines, got %d", len(lines))
}
assertLine(t, []string{"first", "foo", "BAR", "Baz", "Quux", "Blah", "SPtr", "Omit", "garply", "last"}, lines[0])
assertLine(t, []string{"aaa", "f", "1", "baz", "0.2", "2", "*string", "", "3.141592653589793", "zzz"}, lines[1])
}
func Test_writeTo_embedptr_nil(t *testing.T) {
b := bytes.Buffer{}
e := &encoder{out: &b}
s := []EmbedPtrSample{
{},
}
if err := writeTo(NewSafeCSVWriter(csv.NewWriter(e.out)), s, false); err != nil {
t.Fatal(err)
}
lines, err := csv.NewReader(&b).ReadAll()
if err != nil {
t.Fatal(err)
}
if len(lines) != 2 {
t.Fatalf("expected 2 lines, got %d", len(lines))
}
assertLine(t, []string{"first", "foo", "BAR", "Baz", "Quux", "Blah", "SPtr", "Omit", "garply", "last"}, lines[0])
assertLine(t, []string{"", "", "", "", "", "", "", "", "0", ""}, lines[1])
}
func Test_writeTo_embedmarshal(t *testing.T) {
b := bytes.Buffer{}
e := &encoder{out: &b}
s := []EmbedMarshal{
{
Foo: &MarshalSample{Dummy: "bar"},
},
}
if err := writeTo(NewSafeCSVWriter(csv.NewWriter(e.out)), s, false); err != nil {
t.Fatal(err)
}
lines, err := csv.NewReader(&b).ReadAll()
if err != nil {
t.Fatal(err)
}
if len(lines) != 2 {
t.Fatalf("expected 2 lines, got %d", len(lines))
}
assertLine(t, []string{"foo"}, lines[0])
assertLine(t, []string{"bar"}, lines[1])
}
func Test_writeTo_embedmarshalCSV(t *testing.T) {
// First, create our test data
b := new(bytes.Buffer)
e := &encoder{out: b}
s := []*EmbedMarshalCSV{
{
Symbol: "test",
Timestamp: &MarshalCSVSample{
Seconds: 1656460798,
Nanos: 693201614,
},
},
}
// Next, attempt to write our test data to a CSV format
if err := writeTo(NewSafeCSVWriter(csv.NewWriter(e.out)), s, false); err != nil {
t.Fatal(err)
}
// Now, read in the data we just wrote
lines, err := csv.NewReader(b).ReadAll()
if err != nil {
t.Fatal(err)
}
// Finally, verify the structure of the data
if len(lines) != 2 {
t.Fatalf("expected 2 lines, got %d", len(lines))
}
assertLine(t, []string{"symbol", "timestamp"}, lines[0])
assertLine(t, []string{"test", "1656460798.693201614"}, lines[1])
}
func Test_writeTo_complex_embed(t *testing.T) {
b := bytes.Buffer{}
e := &encoder{out: &b}
sptr := "*string"
sfs := []SkipFieldSample{
{
EmbedSample: EmbedSample{
Qux: "aaa",
Sample: Sample{
Foo: "bbb",
Bar: 111,
Baz: "ddd",
Frop: 1.2e22,
Blah: nil,
SPtr: &sptr,
},
Ignore: "eee",
Grault: 0.1,
Quux: "fff",
},
MoreIgnore: "ggg",
Corge: "hhh",
},
}
if err := writeTo(NewSafeCSVWriter(csv.NewWriter(e.out)), sfs, false); err != nil {
t.Fatal(err)
}
lines, err := csv.NewReader(&b).ReadAll()
if err != nil {
t.Fatal(err)
}
if len(lines) != 2 {
t.Fatalf("expected 2 lines, got %d", len(lines))
}
assertLine(t, []string{"first", "foo", "BAR", "Baz", "Quux", "Blah", "SPtr", "Omit", "garply", "last", "abc"}, lines[0])
assertLine(t, []string{"aaa", "bbb", "111", "ddd", "12000000000000000000000", "", "*string", "", "0.1", "fff", "hhh"}, lines[1])
}
func Test_writeTo_complex_inner_struct_embed(t *testing.T) {
b := bytes.Buffer{}
e := &encoder{out: &b}
sfs := []Level0Struct{
{
Level0Field: level1Struct{
Level1Field: level2Struct{
InnerStruct{
BoolIgnoreField0: false,
BoolField1: false,
StringField2: "email1",
},
},
},
},
{
Level0Field: level1Struct{
Level1Field: level2Struct{
InnerStruct{
BoolIgnoreField0: false,
BoolField1: true,
StringField2: "email2",
},
},
},
},
}
if err := writeTo(NewSafeCSVWriter(csv.NewWriter(e.out)), sfs, true); err != nil {
t.Fatal(err)
}
lines, err := csv.NewReader(&b).ReadAll()
if err != nil {
t.Fatal(err)
}
assertLine(t, []string{"false", "email1"}, lines[0])
assertLine(t, []string{"true", "email2"}, lines[1])
}
func Test_writeToChan(t *testing.T) {
b := bytes.Buffer{}
e := &encoder{out: &b}
c := make(chan interface{})
sptr := "*string"
go func() {
for i := 0; i < 100; i++ {
v := Sample{Foo: "f", Bar: i, Baz: "baz" + strconv.Itoa(i), Frop: float64(i), Blah: nil, SPtr: &sptr}
c <- v
}
close(c)
}()
if err := MarshalChan(c, NewSafeCSVWriter(csv.NewWriter(e.out))); err != nil {
t.Fatal(err)
}
lines, err := csv.NewReader(&b).ReadAll()
if err != nil {
t.Fatal(err)
}
if len(lines) != 101 {
t.Fatalf("expected 100 lines, got %d", len(lines))
}
for i, l := range lines {
if i == 0 {
assertLine(t, []string{"foo", "BAR", "Baz", "Quux", "Blah", "SPtr", "Omit"}, l)
continue
}
assertLine(t, []string{"f", strconv.Itoa(i - 1), "baz" + strconv.Itoa(i-1), strconv.FormatFloat(float64(i-1), 'f', -1, 64), "", "*string", ""}, l)
}
}
func Test_MarshalChan_ClosedChannel(t *testing.T) {
b := bytes.Buffer{}
e := &encoder{out: &b}
c := make(chan interface{})
close(c)
if err := MarshalChan(c, NewSafeCSVWriter(csv.NewWriter(e.out))); !errors.Is(err, ErrChannelIsClosed) {
t.Fatal(err)
}
}
// TestRenamedTypes tests for marshaling functions on redefined basic types.
func TestRenamedTypesMarshal(t *testing.T) {
samples := []RenamedSample{
{RenamedFloatUnmarshaler: 1.4, RenamedFloatDefault: 1.5},
{RenamedFloatUnmarshaler: 2.3, RenamedFloatDefault: 2.4},
}
SetCSVWriter(func(out io.Writer) *SafeCSVWriter {
csvout := NewSafeCSVWriter(csv.NewWriter(out))
csvout.Comma = ';'
return csvout
})
// Switch back to default for tests executed after this
defer SetCSVWriter(DefaultCSVWriter)
csvContent, err := MarshalString(&samples)
if err != nil {
t.Fatal(err)
}
if csvContent != "foo;bar\n1,4;1.5\n2,3;2.4\n" {
t.Fatalf("Error marshaling floats with , as separator. Expected \nfoo;bar\n1,4;1.5\n2,3;2.4\ngot:\n%v", csvContent)
}
// Test that errors raised by MarshalCSV are correctly reported
samples = []RenamedSample{
{RenamedFloatUnmarshaler: 4.2, RenamedFloatDefault: 1.5},
}
_, err = MarshalString(&samples)
if _, ok := err.(MarshalError); !ok {
t.Fatalf("Expected UnmarshalError, got %v", err)
}
}
// TestColumnNameStartingWithPartial is a regression test for issue #274: a column name that
// happens to start with "partial" (e.g. "partial_delivery_number") must not be mistaken for the
// "partial" tag option. Previously the option was detected with strings.HasPrefix, so such a
// column name was swallowed and gocsv fell back to using the Go field name instead.
func TestColumnNameStartingWithPartial(t *testing.T) {
type sample struct {
PartialDeliveryNumber uint `csv:"partial_delivery_number"`
}
// Marshal: the header must be the csv tag, not the Go field name.
csvContent, err := MarshalString(&[]sample{{PartialDeliveryNumber: 7}})
if err != nil {
t.Fatal(err)
}
if expected := "partial_delivery_number\n7\n"; csvContent != expected {
t.Fatalf("expected header from csv tag.\nexpected: %q\ngot: %q", expected, csvContent)
}
// Unmarshal: a CSV whose header is the csv tag must populate the field.
var out []sample
if err := Unmarshal(bytes.NewBufferString("partial_delivery_number\n42\n"), &out); err != nil {
t.Fatal(err)
}
if len(out) != 1 || out[0].PartialDeliveryNumber != 42 {
t.Fatalf("expected PartialDeliveryNumber=42, got %+v", out)
}
}
// TestCustomTagSeparatorMarshal tests for custom tag separator in marshalling.
func TestCustomTagSeparatorMarshal(t *testing.T) {
samples := []RenamedSample{
{RenamedFloatUnmarshaler: 1.4, RenamedFloatDefault: 1.5},
{RenamedFloatUnmarshaler: 2.3, RenamedFloatDefault: 2.4},
}
TagSeparator = " | "
// Switch back to default TagSeparator after this
defer func() {
TagSeparator = ","
}()
csvContent, err := MarshalString(&samples)
if err != nil {
t.Fatal(err)
}
if csvContent != "foo|bar\n1,4|1.5\n2,3|2.4\n" {
t.Fatalf("Error marshaling floats with , as separator. Expected \nfoo|bar\n1,4|1.5\n2,3|2.4\ngot:\n%v", csvContent)
}
}
func (rf *RenamedFloat64Unmarshaler) MarshalCSV() (csv string, err error) {
if *rf == RenamedFloat64Unmarshaler(4.2) {
return "", MarshalError{"Test error: Invalid float 4.2"}
}
csv = strconv.FormatFloat(float64(*rf), 'f', 1, 64)
csv = strings.Replace(csv, ".", ",", -1)
return csv, nil
}
type MarshalError struct {
msg string
}
func (e MarshalError) Error() string {
return e.msg
}
func Benchmark_MarshalCSVWithoutHeaders(b *testing.B) {
dst := NewSafeCSVWriter(csv.NewWriter(ioutil.Discard))
for n := 0; n < b.N; n++ {
err := MarshalCSVWithoutHeaders([]Sample{{}}, dst)
if err != nil {
b.Fatal(err)
}
}
}
func Test_writeTo_nested_struct(t *testing.T) {
b := bytes.Buffer{}
e := &encoder{out: &b}
s := []NestedSample{
{
Inner1: InnerStruct{
BoolIgnoreField0: false,
BoolField1: false,
StringField2: "email_one",
},
Inner2: InnerStruct{
BoolIgnoreField0: true,
BoolField1: true,
StringField2: "email_two",
},
InnerIgnore: InnerStruct{
BoolIgnoreField0: true,
BoolField1: false,
StringField2: "email_ignore",
},
Inner3: NestedEmbedSample{InnerStruct{
BoolIgnoreField0: true,
BoolField1: false,
StringField2: "email_three",
}},
},
}
if err := writeTo(NewSafeCSVWriter(csv.NewWriter(e.out)), s, false); err != nil {
t.Fatal(err)
}
lines, err := csv.NewReader(&b).ReadAll()
if err != nil {
t.Fatal(err)
}
if len(lines) != 2 {
t.Fatalf("expected 2 lines, got %d", len(lines))
}
assertLine(t, []string{"one.boolField1", "one.stringField2", "two.boolField1", "two.stringField2", "three.boolField1", "three.stringField2"}, lines[0])
assertLine(t, []string{"false", "email_one", "true", "email_two", "false", "email_three"}, lines[1])
t.Run("struct is a child element ", func(t *testing.T) {
type Nested struct {
UpdatedAt *time.Time `csv:"updated_at"`
}
type Row struct {
Name string `csv:"name"`
Data Nested `csv:"data"`
}
data := time.Date(2022, 12, 11, 13, 14, 15, 123456, time.UTC)
dat := []Row{{Name: "matthew", Data: Nested{UpdatedAt: &data}}}
got, err := MarshalString(dat)
if err != nil {
t.Fatal(err)
}
want := `name,data.updated_at
matthew,2022-12-11T13:14:15.000123456Z
`
if got != want {
t.Fatalf("Error marshaling floats with , as separator. Expected \n%s\ngot:\n%s", want, got)
}
})
}
func Test_non_marshaling_nested_fields_are_prefixed(t *testing.T) {
b := bytes.Buffer{}
e := &encoder{out: &b}
time1 := time.Date(2021, 2, 19, 0, 0, 0, 0, time.UTC)
time2 := time.Date(2022, 2, 19, 0, 0, 0, 0, time.UTC)
time3 := time.Date(2023, 2, 19, 0, 0, 0, 0, time.UTC)
s := []SameNameStruct{
{
Inner2: &InnerStruct2{
Bar: "bar1",
Inner3: InnerStruct3{Bar: "bar2", Foo: time1},
},
Inner3: InnerStruct3{Bar: "bar3", Foo: time2},
Foo: time3,
},
}
if err := writeTo(NewSafeCSVWriter(csv.NewWriter(e.out)), s, false); err != nil {
t.Fatal(err)
}
lines, err := csv.NewReader(&b).ReadAll()
if err != nil {
t.Fatal(err)
}
if len(lines) != 2 {
t.Fatalf("expected 2 lines, got %d", len(lines))
}
// The headers should contain the struct path prefixes even if struct is responsible for its marshalling like time.Time
assertLine(t, []string{"Inner2.Bar", "Inner2.Inner3.Bar", "Inner2.Inner3.Foo", "Inner3.Bar", "Inner3.Foo", "Foo"}, lines[0])
assertLine(t, []string{"bar1", "bar2", "2021-02-19T00:00:00Z", "bar3", "2022-02-19T00:00:00Z", "2023-02-19T00:00:00Z"}, lines[1])
}
func Test_writeTo_inline_nested_struct(t *testing.T) {
b := bytes.Buffer{}
e := &encoder{out: &b}
s := []InlineFooSample{
{Bar: InlineBar{A: 1, B: 2}, X: 3},
}
if err := writeTo(NewSafeCSVWriter(csv.NewWriter(e.out)), s, false); err != nil {
t.Fatal(err)
}
lines, err := csv.NewReader(&b).ReadAll()
if err != nil {
t.Fatal(err)
}
if len(lines) != 2 {
t.Fatalf("expected 2 lines, got %d", len(lines))
}
// The `csv:"."` tag inlines Bar's fields without the "Bar." prefix.
assertLine(t, []string{"a", "b", "x"}, lines[0])
assertLine(t, []string{"1", "2", "3"}, lines[1])
t.Run("pointer to struct", func(t *testing.T) {
got, err := MarshalString([]InlineFooPtrSample{{Bar: &InlineBar{A: 4, B: 5}, X: 6}})
if err != nil {
t.Fatal(err)
}
want := "a,b,x\n4,5,6\n"
if got != want {
t.Fatalf("expected:\n%s\ngot:\n%s", want, got)
}
})
t.Run("inline struct nested under a prefixed field", func(t *testing.T) {
got, err := MarshalString([]InlineOuterSample{{Foo: InlineFooSample{Bar: InlineBar{A: 1, B: 2}, X: 3}, Y: 4}})
if err != nil {
t.Fatal(err)
}
// Foo is prefixed with "foo", and Bar's "." tag drops only its own name,
// so Bar's fields keep the inherited "foo." prefix.
want := "foo.a,foo.b,foo.x,y\n1,2,3,4\n"
if got != want {
t.Fatalf("expected:\n%s\ngot:\n%s", want, got)
}
})
}
+132
View File
@@ -0,0 +1,132 @@
package main
import (
"encoding/csv"
"fmt"
"io"
"io/ioutil"
"os"
"time"
"github.com/gocarina/gocsv"
)
type NotUsed struct {
Name string
}
type Client struct { // Our example struct, you can use "-" to ignore a field
ID string `csv:"client_id"`
Name string `csv:"client_name"`
Age string `csv:"client_age"`
NotUsedString string `csv:"-"`
NotUsedStruct NotUsed `csv:"-"`
Address1 Address `csv:"addr1"`
Address2 Address //`csv:"addr2"` will use Address2 in header
Employed DateTime `csv:"employed"`
}
type Address struct {
Street string `csv:"street"`
City string `csv:"city"`
}
var _ gocsv.TypeMarshaller = new(DateTime)
var _ gocsv.TypeUnmarshaller = new(DateTime)
type DateTime struct {
time.Time
}
// Convert the internal date as CSV string
func (date *DateTime) MarshalCSV() (string, error) {
return date.String(), nil
}
// You could also use the standard Stringer interface
func (date DateTime) String() string {
return date.Time.Format("20060102")
}
// Convert the CSV string as internal date
func (date *DateTime) UnmarshalCSV(csv string) (err error) {
date.Time, err = time.Parse("20060102", csv)
return err
}
func main() {
// set the pipe as the delimiter for writing
gocsv.TagSeparator = "|"
// set the pipe as the delimiter for reading
gocsv.SetCSVReader(func(in io.Reader) gocsv.CSVReader {
r := csv.NewReader(in)
r.Comma = '|'
return r
})
// Create an empty clients file
clientsFile, err := os.OpenFile("clients.csv", os.O_RDWR|os.O_CREATE|os.O_TRUNC, os.ModePerm)
if err != nil {
panic(err)
}
defer clientsFile.Close()
// Create clients
clients := []*Client{
{ID: "12", Name: "John", Age: "21",
Address1: Address{"Street 1", "City1"},
Address2: Address{"Street 2", "City2"},
Employed: DateTime{time.Date(2022, 11, 04, 12, 0, 0, 0, time.UTC)},
},
{ID: "13", Name: "Fred",
Address1: Address{`Main "Street" 1`, "City1"}, // show quotes in value
Address2: Address{"Main Street 2", "City2"},
Employed: DateTime{time.Date(2022, 11, 04, 13, 0, 0, 0, time.UTC)},
},
{ID: "14", Name: "James", Age: "32",
Address1: Address{"Center Street 1", "City1"},
Address2: Address{"Center Street 2", "City2"},
Employed: DateTime{time.Date(2022, 11, 04, 14, 0, 0, 0, time.UTC)},
},
{ID: "15", Name: "Danny",
Address1: Address{"State Street 1", "City1"},
Address2: Address{"State Street 2", "City2"},
Employed: DateTime{time.Date(2022, 11, 04, 15, 0, 0, 0, time.UTC)},
},
}
// Save clients to csv file
if err = gocsv.MarshalFile(&clients, clientsFile); err != nil {
panic(err)
}
// Reset the file reader
if _, err := clientsFile.Seek(0, io.SeekStart); err != nil {
panic(err)
}
// Read file and print to console
b, err := ioutil.ReadAll(clientsFile)
if err != nil {
panic(err)
}
fmt.Println("clients.csv:")
fmt.Println(string(b))
// Reset the file reader
if _, err := clientsFile.Seek(0, io.SeekStart); err != nil {
panic(err)
}
// Load clients from file
var newClients []*Client
if err := gocsv.UnmarshalFile(clientsFile, &newClients); err != nil {
panic(err)
}
fmt.Println("clients:")
for _, c := range newClients {
fmt.Printf("%s:%s Adress1:%s, %s Address2:%s, %s Employed: %s\n",
c.ID, c.Name,
c.Address1.Street, c.Address1.City,
c.Address2.Street, c.Address2.City,
c.Employed,
)
}
}
+18
View File
@@ -0,0 +1,18 @@
package main
func Example() {
main()
// Output:
// clients.csv:
// client_id|client_name|client_age|addr1.street|addr1.city|Address2.street|Address2.city|employed
// 12|John|21|Street 1|City1|Street 2|City2|20221104
// 13|Fred||"Main ""Street"" 1"|City1|Main Street 2|City2|20221104
// 14|James|32|Center Street 1|City1|Center Street 2|City2|20221104
// 15|Danny||State Street 1|City1|State Street 2|City2|20221104
//
// clients:
// 12:John Adress1:Street 1, City1 Address2:Street 2, City2 Employed: 20221104
// 13:Fred Adress1:Main "Street" 1, City1 Address2:Main Street 2, City2 Employed: 20221104
// 14:James Adress1:Center Street 1, City1 Address2:Center Street 2, City2 Employed: 20221104
// 15:Danny Adress1:State Street 1, City1 Address2:State Street 2, City2 Employed: 20221104
}
+287
View File
@@ -0,0 +1,287 @@
package gocsv
import (
"fmt"
"reflect"
"strconv"
"strings"
"sync"
)
// --------------------------------------------------------------------------
// Reflection helpers
type structInfo struct {
Fields []fieldInfo
Err error
}
// fieldInfo is a struct field that should be mapped to a CSV column, or vice-versa
// Each IndexChain element before the last is the index of an the embedded struct field
// that defines Key as a tag
type fieldInfo struct {
keys []string
omitEmpty bool
IndexChain []int
defaultValue string
partial bool
inline bool
}
func (f fieldInfo) getFirstKey() string {
return f.keys[0]
}
func (f fieldInfo) matchesKey(key string) bool {
for _, k := range f.keys {
if key == k || strings.TrimSpace(key) == k || (f.partial && strings.Contains(key, k)) || removeZeroWidthChars(key) == k {
return true
}
}
return false
}
// zwchs is Zero Width Characters map
var zwchs = map[rune]struct{}{
'\u200B': {}, // zero width space (U+200B)
'\uFEFF': {}, // zero width no-break space (U+FEFF)
'\u200D': {}, // zero width joiner (U+200D)
'\u200C': {}, // zero width non-joiner (U+200C)
}
func removeZeroWidthChars(s string) string {
return strings.Map(func(r rune) rune {
if _, ok := zwchs[r]; ok {
return -1
}
return r
}, s)
}
var structMap = make(map[reflect.Type]*structInfo)
var structMapMutex sync.RWMutex
func getStructInfo(rType reflect.Type) *structInfo {
structInfos := &loadHeaderNormalization().structInfos
stInfo, ok := structInfos.Load(rType)
if ok {
return stInfo.(*structInfo)
}
fieldsList, err := getFieldInfos(rType, []int{}, []string{}, make(map[reflect.Type]bool))
stInfo = &structInfo{Fields: fieldsList, Err: err}
structInfos.Store(rType, stInfo)
return stInfo.(*structInfo)
}
func getFieldInfos(rType reflect.Type, parentIndexChain []int, parentKeys []string, ancestors map[reflect.Type]bool) ([]fieldInfo, error) {
if ancestors[rType] {
return nil, fmt.Errorf("cyclic struct type %s: exclude the recursive field with csv:\"-\"", rType)
}
ancestors[rType] = true
defer delete(ancestors, rType)
fieldsCount := rType.NumField()
fieldsList := make([]fieldInfo, 0, fieldsCount)
for i := 0; i < fieldsCount; i++ {
field := rType.Field(i)
if field.PkgPath != "" {
continue
}
var cpy = make([]int, len(parentIndexChain))
copy(cpy, parentIndexChain)
indexChain := append(cpy, i)
// Determine whether it's a struct we can expand into nested columns.
// Structs that implement any of the text or CSV marshaling methods
// should result in one value and not have their fields exposed
fieldType := field.Type
if fieldType.Kind() == reflect.Ptr {
fieldType = fieldType.Elem()
}
isExpandableStruct := fieldType.Kind() == reflect.Struct && !canMarshal(fieldType)
var currFieldInfo *fieldInfo
if !field.Anonymous {
filteredTags := []string{}
currFieldInfo, filteredTags = filterTags(TagName, indexChain, field)
if len(filteredTags) == 1 && filteredTags[0] == "-" {
// ignore nested structs with - tag
continue
} else if isExpandableStruct && len(filteredTags) == 1 && filteredTags[0] == "." {
// inline nested structs with . tag
currFieldInfo.inline = true
currFieldInfo.keys = parentKeys
} else if len(filteredTags) > 0 && filteredTags[0] != "" {
currFieldInfo.keys = filteredTags
} else {
currFieldInfo.keys = []string{normalizeName(field.Name)}
}
if len(parentKeys) > 0 && currFieldInfo != nil && !currFieldInfo.inline {
// create cartesian product of keys
// eg: parent keys x field keys
keys := make([]string, 0, len(parentKeys)*len(currFieldInfo.keys))
for _, pkey := range parentKeys {
for _, ckey := range currFieldInfo.keys {
keys = append(keys, normalizeName(fmt.Sprintf("%s%s%s", pkey, FieldsCombiner, ckey)))
}
currFieldInfo.keys = keys
}
}
}
// handle struct
if isExpandableStruct {
// if the field is an embedded struct, pass along parent keys
keys := parentKeys
if currFieldInfo != nil {
keys = currFieldInfo.keys
}
childFields, err := getFieldInfos(fieldType, indexChain, keys, ancestors)
if err != nil {
return nil, err
}
fieldsList = append(fieldsList, childFields...)
continue
}
// if the field is an embedded struct, ignore the csv tag
if currFieldInfo == nil {
continue
}
if field.Type.Kind() == reflect.Slice || field.Type.Kind() == reflect.Array {
var arrayLength = -1
// if the field is a slice or an array, see if it has a `csv[n]` tag
if arrayTag, ok := field.Tag.Lookup(TagName + "[]"); ok {
arrayLength, _ = strconv.Atoi(arrayTag)
}
// slices or arrays of Struct get special handling
if field.Type.Elem().Kind() == reflect.Struct {
fieldInfos, err := getFieldInfos(field.Type.Elem(), []int{}, []string{}, ancestors)
if err != nil {
return nil, err
}
// if no special csv[] tag was supplied, just include the field directly
if arrayLength == -1 {
fieldsList = append(fieldsList, *currFieldInfo)
} else {
// When the field is a slice/array of structs, create a fieldInfo for each index and each field
for idx := 0; idx < arrayLength; idx++ {
// copy index chain and append array index
var cpy2 = make([]int, len(indexChain))
copy(cpy2, indexChain)
arrayIndexChain := append(cpy2, idx)
for _, childFieldInfo := range fieldInfos {
// copy array index chain and append array index
var cpy3 = make([]int, len(arrayIndexChain))
copy(cpy3, arrayIndexChain)
arrayFieldInfo := fieldInfo{
IndexChain: append(cpy3, childFieldInfo.IndexChain...),
omitEmpty: childFieldInfo.omitEmpty,
defaultValue: childFieldInfo.defaultValue,
partial: childFieldInfo.partial,
}
// create cartesian product of keys
// eg: array field keys x struct field keys
for _, akey := range currFieldInfo.keys {
for _, fkey := range childFieldInfo.keys {
arrayFieldInfo.keys = append(arrayFieldInfo.keys, normalizeName(fmt.Sprintf("%s[%d].%s", akey, idx, fkey)))
}
}
fieldsList = append(fieldsList, arrayFieldInfo)
}
}
}
} else if arrayLength > 0 {
// When the field is a slice/array of primitives, create a fieldInfo for each index
for idx := 0; idx < arrayLength; idx++ {
// copy index chain and append array index
var cpy2 = make([]int, len(indexChain))
copy(cpy2, indexChain)
arrayFieldInfo := fieldInfo{
IndexChain: append(cpy2, idx),
omitEmpty: currFieldInfo.omitEmpty,
defaultValue: currFieldInfo.defaultValue,
partial: currFieldInfo.partial,
}
for _, akey := range currFieldInfo.keys {
arrayFieldInfo.keys = append(arrayFieldInfo.keys, normalizeName(fmt.Sprintf("%s[%d]", akey, idx)))
}
fieldsList = append(fieldsList, arrayFieldInfo)
}
} else {
fieldsList = append(fieldsList, *currFieldInfo)
}
} else {
fieldsList = append(fieldsList, *currFieldInfo)
}
}
return fieldsList, nil
}
func filterTags(tagName string, indexChain []int, field reflect.StructField) (*fieldInfo, []string) {
currFieldInfo := fieldInfo{IndexChain: indexChain}
fieldTag := field.Tag.Get(tagName)
fieldTags := strings.Split(fieldTag, TagSeparator)
filteredTags := []string{}
for _, fieldTagEntry := range fieldTags {
trimmedFieldTagEntry := strings.TrimSpace(fieldTagEntry) // handles cases like `csv:"foo, omitempty, default=test"`
if trimmedFieldTagEntry == "omitempty" {
currFieldInfo.omitEmpty = true
} else if trimmedFieldTagEntry == "partial" {
// Must be an exact match: using HasPrefix here would mistake a column name that
// happens to start with "partial" (e.g. `csv:"partial_delivery_number"`) for the
// "partial" option, dropping the real column name. See issue #274.
currFieldInfo.partial = true
} else if strings.HasPrefix(trimmedFieldTagEntry, "default=") {
currFieldInfo.defaultValue = strings.TrimPrefix(trimmedFieldTagEntry, "default=")
} else {
filteredTags = append(filteredTags, normalizeName(trimmedFieldTagEntry))
}
}
return &currFieldInfo, filteredTags
}
func getConcreteContainerInnerType(in reflect.Type) (inInnerWasPointer bool, inInnerType reflect.Type) {
inInnerType = in.Elem()
inInnerWasPointer = false
if inInnerType.Kind() == reflect.Ptr {
inInnerWasPointer = true
inInnerType = inInnerType.Elem()
}
return inInnerWasPointer, inInnerType
}
func getConcreteReflectValueAndType(in interface{}) (reflect.Value, reflect.Type) {
value := reflect.ValueOf(in)
if value.Kind() == reflect.Ptr {
value = value.Elem()
}
return value, value.Type()
}
var errorInterface = reflect.TypeOf((*error)(nil)).Elem()
func isErrorType(outType reflect.Type) bool {
if outType.Kind() != reflect.Interface {
return false
}
return outType.Implements(errorInterface)
}
+5 -2
View File
@@ -130,7 +130,9 @@ func toInt(in interface{}) (int64, error) {
return 0, nil return 0, nil
} }
out := strings.SplitN(s, ".", 2) out := strings.SplitN(s, ".", 2)
return strconv.ParseInt(out[0], 0, 64) // base 10, not 0: a CSV field is data, not Go source, so "010" is ten
// and "09" is nine rather than an invalid octal literal
return strconv.ParseInt(out[0], 10, 64)
case reflect.Bool: case reflect.Bool:
if inValue.Bool() { if inValue.Bool() {
return 1, nil return 1, nil
@@ -164,7 +166,8 @@ func toUint(in interface{}) (uint64, error) {
} }
return uint64(f), nil return uint64(f), nil
} }
return strconv.ParseUint(s, 0, 64) // base 10, see toInt
return strconv.ParseUint(s, 10, 64)
case reflect.Bool: case reflect.Bool:
if inValue.Bool() { if inValue.Bool() {
return 1, nil return 1, nil
+55
View File
@@ -139,3 +139,58 @@ func TestToInt(t *testing.T) {
} }
} }
} }
func TestToIntLeadingZeros(t *testing.T) {
testCases := []struct {
field string
result int64
}{
{"08", 8},
{"09", 9},
{"010", 10},
{"007", 7},
{"0009", 9},
{"-08", -8},
{"-010", -10},
{"0", 0},
{"00", 0},
{"010.9", 10},
}
for _, item := range testCases {
out, err := toInt(item.field)
if err != nil {
t.Errorf("toInt(%q) returned an error: %s", item.field, err)
continue
}
if out != item.result {
t.Errorf("toInt(%q) = %d, want %d", item.field, out, item.result)
}
}
}
func TestToUintLeadingZeros(t *testing.T) {
testCases := []struct {
field string
result uint64
}{
{"08", 8},
{"09", 9},
{"010", 10},
{"007", 7},
{"0009", 9},
{"0", 0},
{"00", 0},
}
for _, item := range testCases {
out, err := toUint(item.field)
if err != nil {
t.Errorf("toUint(%q) returned an error: %s", item.field, err)
continue
}
if out != item.result {
t.Errorf("toUint(%q) = %d, want %d", item.field, out, item.result)
}
}
}