Add supported encoding

Use htmlindex.Get to get the encoding.

Author: onozaty

cmd/header_test.go

@@ -201,13 +201,13 @@ func TestHeaderCmd_encoding_shift_jis(t *testing.T) {
 	}
 }
 
-func TestHeaderCmd_encoding_eucjp(t *testing.T) {
+func TestHeaderCmd_encoding_euc_jp(t *testing.T) {
 
 	rootCmd := newRootCmd()
 	rootCmd.SetArgs([]string{
 		"header",
 		"-i", "../testdata/users-eucjp.csv",
-		"--encoding", "eucjp",
+		"--encoding", "EUC-JP",
 	})
 
 	buf := new(bytes.Buffer)
@@ -225,13 +225,13 @@ func TestHeaderCmd_encoding_eucjp(t *testing.T) {
 	}
 }
 
-func TestHeaderCmd_encoding_euc_jp(t *testing.T) {
+func TestHeaderCmd_encoding_koi8_r(t *testing.T) {
 
 	rootCmd := newRootCmd()
 	rootCmd.SetArgs([]string{
 		"header",
-		"-i", "../testdata/users-eucjp.csv",
-		"--encoding", "EUC-JP",
+		"-i", "../testdata/users-koi8r.csv",
+		"--encoding", "koi8-r",
 	})
 
 	buf := new(bytes.Buffer)
@@ -244,7 +244,55 @@ func TestHeaderCmd_encoding_euc_jp(t *testing.T) {
 
 	result := buf.String()
 
-	if result != "ID\n名前\n年齢\n" {
+	if result != "ID\nНазовите\nвозраст\n" {
+		t.Fatal("failed test\n", result)
+	}
+}
+
+func TestHeaderCmd_encoding_euc_kr(t *testing.T) {
+
+	rootCmd := newRootCmd()
+	rootCmd.SetArgs([]string{
+		"header",
+		"-i", "../testdata/users-euckr.csv",
+		"--encoding", "euc-kr",
+	})
+
+	buf := new(bytes.Buffer)
+	rootCmd.SetOutput(buf)
+
+	err := rootCmd.Execute()
+	if err != nil {
+		t.Fatal("failed test\n", err)
+	}
+
+	result := buf.String()
+
+	if result != "ID\n이름\n나이\n" {
+		t.Fatal("failed test\n", result)
+	}
+}
+
+func TestHeaderCmd_encoding_big5(t *testing.T) {
+
+	rootCmd := newRootCmd()
+	rootCmd.SetArgs([]string{
+		"header",
+		"-i", "../testdata/users-big5.csv",
+		"--encoding", "big5",
+	})
+
+	buf := new(bytes.Buffer)
+	rootCmd.SetOutput(buf)
+
+	err := rootCmd.Execute()
+	if err != nil {
+		t.Fatal("failed test\n", err)
+	}
+
+	result := buf.String()
+
+	if result != "ID\n名稱\n年齡\n" {
 		t.Fatal("failed test\n", result)
 	}
 }

cmd/root.go

@@ -20,7 +20,7 @@ func newRootCmd() *cobra.Command {
 	rootCmd.PersistentFlags().StringP("quote", "", "", "(optional) CSV quote. The default is '\"'")
 	rootCmd.PersistentFlags().StringP("sep", "", "", "(optional) CSV record separator. The default is CRLF.")
 	rootCmd.PersistentFlags().BoolP("allquote", "", false, "(optional) Always quote CSV fields. The default is to quote only the necessary fields.")
-	rootCmd.PersistentFlags().StringP("encoding", "", "", "(optional) CSV encoding. The default is utf-8. Supported encodings: utf-8, shift_jis, euc-jp")
+	rootCmd.PersistentFlags().StringP("encoding", "", "", "(optional) CSV encoding. The default is utf-8.")
 	rootCmd.PersistentFlags().BoolP("bom", "", false, "(optional) CSV with BOM. When reading, the BOM will be automatically removed without this flag.")
 	rootCmd.PersistentFlags().SortFlags = false
 	rootCmd.Flags().SortFlags = false

cmd/transform.go

@@ -45,7 +45,7 @@ func newTransformCmd() *cobra.Command {
 	transformCmd.Flags().StringP("out-quote", "", "", "(optional) Output CSV quote. The default is '\"'")
 	transformCmd.Flags().StringP("out-sep", "", "", "(optional) Output CSV record separator. The default is CRLF.")
 	transformCmd.Flags().BoolP("out-allquote", "", false, "(optional) Always quote output CSV fields. The default is to quote only the necessary fields.")
-	transformCmd.Flags().StringP("out-encoding", "", "", "(optional) Output CSV encoding. The default is utf-8. Supported encodings: utf-8, shift_jis, euc-jp")
+	transformCmd.Flags().StringP("out-encoding", "", "", "(optional) Output CSV encoding. The default is utf-8.")
 	transformCmd.Flags().BoolP("out-bom", "", false, "(optional) Output CSV with BOM.")
 
 	return transformCmd

csv/csv.go

@@ -6,8 +6,9 @@ import (
 	"strings"
 
 	"github.com/onozaty/go-customcsv"
+	"golang.org/x/exp/slices"
 	"golang.org/x/text/encoding"
-	"golang.org/x/text/encoding/japanese"
+	"golang.org/x/text/encoding/htmlindex"
 	"golang.org/x/text/transform"
 )
 
@@ -78,18 +79,15 @@ func NewCsvWriter(w io.Writer, f Format) CsvWriter {
 
 func Encoding(name string) (encoding.Encoding, error) {
 
-	normalizeName := strings.ReplaceAll(strings.ReplaceAll(strings.ToLower(name), "_", ""), "-", "")
-
-	switch normalizeName {
-	case "utf8":
+	if slices.Contains([]string{"utf8", "utf-8"}, strings.ToLower(name)) {
 		// UTF-8の場合は変換不要
 		return nil, nil
-	case "shiftjis", "sjis":
-		return japanese.ShiftJIS, nil
-	case "eucjp":
-		return japanese.EUCJP, nil
-	default:
+	}
+
+	encoding, err := htmlindex.Get(name)
+	if err != nil {
 		return nil, fmt.Errorf("invalid encoding name: %s", name)
 	}
 
+	return encoding, nil
 }

csv/csv_test.go

@@ -299,7 +299,7 @@ func TestEncoding_sjis(t *testing.T) {
 
 func TestEncoding_eucjp(t *testing.T) {
 
-	encoding1, err := Encoding("eucjp")
+	encoding1, err := Encoding("euc-jp")
 	if err != nil {
 		t.Fatal("failed test\n", err)
 	}

testdata/users-big5.csv

@@ -0,0 +1,3 @@
+ID,�W��,�~��
+1,"Taro, Yamada",20
+2,Hanako,21

testdata/users-euckr.csv

@@ -0,0 +1,3 @@
+ID,�̸�,����
+1,"Taro, Yamada",20
+2,Hanako,21

testdata/users-koi8r.csv

@@ -0,0 +1,3 @@
+ID,��������,�������
+1,"Taro, Yamada",20
+2,Hanako,21