# This file is a part of Julia. License is MIT: https://julialang.org/license

using Test, Random
using DelimitedFiles

isequaldlm(m1, m2, t) = isequal(m1, m2) && (eltype(m1) == eltype(m2) == t)

@testset "readdlm" begin
    @test isequaldlm(readdlm(IOBuffer("1\t2\n3\t4\n5\t6\n")), [1. 2; 3 4; 5 6], Float64)
    @test isequaldlm(readdlm(IOBuffer("1\t2\n3\t4\n5\t6\n"), Int), [1 2; 3 4; 5 6], Int)
    @test isequaldlm(readdlm(IOBuffer("1,22222222222222222222222222222222222222,0x3,10e6\n2000.1,true,false,-10.34"), ',', Any),
        reshape(Any[1,2000.1,Float64(22222222222222222222222222222222222222),true,0x3,false,10e6,-10.34], 2, 4), Any)

    @test isequaldlm(readdlm(IOBuffer("-9223355253176920979,9223355253176920979"), ',', Int64), Int64[-9223355253176920979  9223355253176920979], Int64)

    @test size(readdlm(IOBuffer("1,2,3,4"), ',')) == (1,4)
    @test size(readdlm(IOBuffer("1,2,3,"), ',')) == (1,4)
    @test size(readdlm(IOBuffer("1,2,3,4\n"), ',')) == (1,4)
    @test size(readdlm(IOBuffer("1,2,3,\n"), ',')) == (1,4)
    @test size(readdlm(IOBuffer("1,2,3,4\n1,2,3,4"), ',')) == (2,4)
    @test size(readdlm(IOBuffer("1,2,3,4\n1,2,3,"), ',')) == (2,4)
    @test size(readdlm(IOBuffer("1,2,3,4\n1,2,3"), ',')) == (2,4)

    @test size(readdlm(IOBuffer("1,2,3,4\r\n"), ',')) == (1,4)
    @test size(readdlm(IOBuffer("1,2,3,4\r\n1,2,3\r\n"), ',')) == (2,4)
    @test size(readdlm(IOBuffer("1,2,3,4\r\n1,2,3,4\r\n"), ',')) == (2,4)
    @test size(readdlm(IOBuffer("1,2,3,\"4\"\r\n1,2,3,4\r\n"), ',')) == (2,4)

    @test size(readdlm(IOBuffer("1 2 3 4\n1 2 3"))) == (2,4)
    @test size(readdlm(IOBuffer("1\t2 3 4\n1 2 3"))) == (2,4)
    @test size(readdlm(IOBuffer("1\t 2 3 4\n1 2 3"))) == (2,4)
    @test size(readdlm(IOBuffer("1\t 2 3 4\n1 2 3\n"))) == (2,4)
    @test size(readdlm(IOBuffer("1,,2,3,4\n1,2,3\n"), ',')) == (2,5)

    let result1 = reshape(Any["", "", "", "", "", "", 1.0, 1.0, "", "", "", "", "", 1.0, 2.0, "", 3.0, "", "", "", "", "", 4.0, "", "", ""], 2, 13),
        result2 = reshape(Any[1.0, 1.0, 2.0, 1.0, 3.0, "", 4.0, ""], 2, 4)

        @test isequaldlm(readdlm(IOBuffer(",,,1,,,,2,3,,,4,\n,,,1,,,1\n"), ','), result1, Any)
        @test isequaldlm(readdlm(IOBuffer("   1    2 3   4 \n   1   1\n")), result2, Any)
        @test isequaldlm(readdlm(IOBuffer("   1    2 3   4 \n   1   1\n"), ' '), result1, Any)
        @test isequaldlm(readdlm(IOBuffer("1 2\n3 4 \n")), [[1.0, 3.0] [2.0, 4.0]], Float64)
    end

    let result1 = reshape(Any["", "", "", "", "", "", "भारत", 1.0, "", "", "", "", "", 1.0, 2.0, "", 3.0, "", "", "", "", "", 4.0, "", "", ""], 2, 13)
        @test isequaldlm(readdlm(IOBuffer(",,,भारत,,,,2,3,,,4,\n,,,1,,,1\n"), ',') , result1, Any)
    end

    let result1 = reshape(Any[1.0, 1.0, 2.0, 2.0, 3.0, 3.0, 4.0, ""], 2, 4)
        @test isequaldlm(readdlm(IOBuffer("1\t 2 3 4\n1 2 3")), result1, Any)
        @test isequaldlm(readdlm(IOBuffer("1\t 2 3 4\n1 2 3 ")), result1, Any)
        @test isequaldlm(readdlm(IOBuffer("1\t 2 3 4\n1 2 3\n")), result1, Any)
        @test isequaldlm(readdlm(IOBuffer("1,2,3,4\n1,2,3\n"), ','), result1, Any)
        @test isequaldlm(readdlm(IOBuffer("1,2,3,4\n1,2,3"), ','), result1, Any)
        @test isequaldlm(readdlm(IOBuffer("1,2,3,4\r\n1,2,3\r\n"), ','), result1, Any)
        @test isequaldlm(readdlm(IOBuffer("1,2,3,\"4\"\r\n1,2,3\r\n"), ','), result1, Any)
    end

    let result1 = reshape(Any["abc", "hello", "def,ghi", " \"quote\" ", "new\nline", "world"], 2, 3),
        result2 = reshape(Any["abc", "line\"", "\"hello\"", "\"def", "", "\" \"\"quote\"\" \"", "ghi\"", "", "world", "\"new", "", ""], 3, 4)

        @test isequaldlm(readdlm(IOBuffer("abc,\"def,ghi\",\"new\nline\"\n\"hello\",\" \"\"quote\"\" \",world"), ','), result1, Any)
        @test isequaldlm(readdlm(IOBuffer("abc,\"def,ghi\",\"new\nline\"\n\"hello\",\" \"\"quote\"\" \",world"), ',', quotes=false), result2, Any)
    end

    let result1 = reshape(Any["t", "c", "", "c"], 2, 2),
        result2 = reshape(Any["t", "\"c", "t", "c"], 2, 2)
        @test isequaldlm(readdlm(IOBuffer("t  \n\"c\" c")), result1, Any)
        @test isequaldlm(readdlm(IOBuffer("t t \n\"\"\"c\" c")), result2, Any)
    end

    @test isequaldlm(readdlm(IOBuffer("\n1,2,3\n4,5,6\n\n\n"), ',', skipblanks=false),
                     reshape(Any["",1.0,4.0,"","","",2.0,5.0,"","","",3.0,6.0,"",""], 5, 3), Any)
    @test isequaldlm(readdlm(IOBuffer("\n1,2,3\n4,5,6\n\n\n"), ',', skipblanks=true), reshape([1.0,4.0,2.0,5.0,3.0,6.0], 2, 3), Float64)
    @test isequaldlm(readdlm(IOBuffer("1,2\n\n4,5"), ',', skipblanks=false), reshape(Any[1.0,"",4.0,2.0,"",5.0], 3, 2), Any)
    @test isequaldlm(readdlm(IOBuffer("1,2\n\n4,5"), ',', skipblanks=true), reshape([1.0,4.0,2.0,5.0], 2, 2), Float64)

    let x = bitrand(5, 10), io = IOBuffer()
        writedlm(io, x)
        seek(io, 0)
        @test readdlm(io, Bool) == x
    end

    let x = [1,2,3], y = [4,5,6], io = IOBuffer()
        writedlm(io, zip(x,y), ",  ")
        seek(io, 0)
        @test readdlm(io, ',') == [x y]
    end

    let x = [0.1 0.3 0.5], io = IOBuffer()
        writedlm(io, x, ", ")
        seek(io, 0)
        @test read(io, String) == "0.1, 0.3, 0.5\n"
    end

    let x = [0.1 0.3 0.5], io = IOBuffer()
        writedlm(io, x, ", ")
        seek(io, 0)
        @test readdlm(io, ',') == [0.1 0.3 0.5]
    end

    let x = ["abc", "def\"ghi", "jk\nl"], y = [1, ",", "\"quoted\""], io = IOBuffer()
        writedlm(io, zip(x,y), ',')
        seek(io, 0)
        @test readdlm(io, ',') == [x y]
    end

    let x = ["a" "b"; "d" ""], io = IOBuffer()
        writedlm(io, x)
        seek(io, 0)
        @test readdlm(io) == x
    end

    let x = ["\"hello\"", "world\""], io = IOBuffer()
        writedlm(io, x, quotes=false)
        @test String(take!(io)) == "\"hello\"\nworld\"\n"

        writedlm(io, x)
        @test String(take!(io)) == "\"\"\"hello\"\"\"\n\"world\"\"\"\n"
    end
end

@testset "comments" begin
    @test isequaldlm(readdlm(IOBuffer("#this is comment\n1,2,3\n#one more comment\n4,5,6"), ',', comments=true), [1. 2. 3.;4. 5. 6.], Float64)
    @test isequaldlm(readdlm(IOBuffer("#this is \n#comment\n1,2,3\n#one more \n#comment\n4,5,6"), ',', comments=true), [1. 2. 3.;4. 5. 6.], Float64)
    @test isequaldlm(readdlm(IOBuffer("1,2,#3\n4,5,6"), ',', comments=true), [1. 2. "";4. 5. 6.], Any)
    @test isequaldlm(readdlm(IOBuffer("1#,2,3\n4,5,6"), ',', comments=true), [1. "" "";4. 5. 6.], Any)
    @test isequaldlm(readdlm(IOBuffer("1,2,\"#3\"\n4,5,6"), ',', comments=true), [1. 2. "#3";4. 5. 6.], Any)
    @test isequaldlm(readdlm(IOBuffer("1,2,3\n #with leading whitespace\n4,5,6"), ',', comments=true), [1. 2. 3.;" " "" "";4. 5. 6.], Any)
end

@testset "without comments" begin
    @test isequaldlm(readdlm(IOBuffer("1,2,#3\n4,5,6"), ','), [1. 2. "#3";4. 5. 6.], Any)
    @test isequaldlm(readdlm(IOBuffer("1#,2,3\n4,5,6"), ','), ["1#" 2. 3.;4. 5. 6.], Any)
    @test isequaldlm(readdlm(IOBuffer("1,2,\"#3\"\n4,5,6"), ','), [1. 2. "#3";4. 5. 6.], Any)
end

@testset "skipstart" begin
    x = ["a" "b" "c"; "d" "e" "f"; "g" "h" "i"; "A" "B" "C"; 1 2 3; 4 5 6; 7 8 9]
    io = IOBuffer()

    writedlm(io, x, quotes=false)
    seek(io, 0)
    (data, hdr) = readdlm(io, header=true, skipstart=3)
    @test data == [1 2 3; 4 5 6; 7 8 9]
    @test hdr == ["A" "B" "C"]

    x = ["a" "b" "\nc"; "d" "\ne" "f"; "g" "h" "i\n"; "A" "B" "C"; 1 2 3; 4 5 6; 7 8 9]
    io = IOBuffer()

    writedlm(io, x, quotes=true)
    seek(io, 0)
    (data, hdr) = readdlm(io, header=true, skipstart=6)
    @test data == [1 2 3; 4 5 6; 7 8 9]
    @test hdr == ["A" "B" "C"]

    io = IOBuffer()
    writedlm(io, x, quotes=false)
    seek(io, 0)
    (data, hdr) = readdlm(io, header=true, skipstart=6)
    @test data == [1 2 3; 4 5 6; 7 8 9]
    @test hdr == ["A" "B" "C"]
end

@testset "i18n" begin
    # source: http://www.i18nguy.com/unicode/unicode-example-utf8.zip
    let i18n_data = ["Origin (English)", "Name (English)", "Origin (Native)", "Name (Native)",
            "Australia", "Nicole Kidman", "Australia", "Nicole Kidman",
            "Austria", "Johann Strauss", "Österreich", "Johann Strauß",
            "Belgium (Flemish)", "Rene Magritte", "België", "René Magritte",
            "Belgium (French)", "Rene Magritte", "Belgique", "René Magritte",
            "Belgium (German)", "Rene Magritte", "Belgien", "René Magritte",
            "Bhutan", "Gonpo Dorji", "འབྲུག་ཡུལ།", "མགོན་པོ་རྡོ་རྗེ།",
            "Canada", "Celine Dion", "Canada", "Céline Dion",
            "Canada - Nunavut (Inuktitut)", "Susan Aglukark", "ᓄᓇᕗᒻᒥᐅᑦ", "ᓱᓴᓐ ᐊᒡᓗᒃᑲᖅ",
            "Democratic People's Rep. of Korea", "LEE Sol-Hee", "조선 민주주의 인민 공화국", "이설희",
            "Denmark", "Soren Hauch-Fausboll", "Danmark", "Søren Hauch-Fausbøll",
            "Denmark", "Soren Kierkegaard", "Danmark", "Søren Kierkegård",
            "Egypt", "Abdel Halim Hafez", "ﻣﺼﺮ", "ﻋﺑﺪﺍﻠﺣﻟﻳﻢ ﺤﺎﻓﻅ",
            "Egypt", "Om Kolthoum", "ﻣﺼﺮ", "ﺃﻡ ﻛﻟﺛﻭﻡ",
            "Eritrea", "Berhane Zeray", "ብርሃነ ዘርኣይ", "ኤርትራ",
            "Ethiopia", "Haile Gebreselassie", "ኃይሌ ገብረሥላሴ", "ኢትዮጵያ",
            "France", "Gerard Depardieu", "France", "Gérard Depardieu",
            "France", "Jean Reno", "France", "Jean Réno",
            "France", "Camille Saint-Saens", "France", "Camille Saint-Saëns",
            "France", "Mylene Demongeot", "France", "Mylène Demongeot",
            "France", "Francois Truffaut", "France", "François Truffaut",
            "France (Braille)", "Louis Braille", "⠋⠗⠁⠝⠉⠑", "⠇⠕⠥⠊⠎⠀<BR>⠃⠗⠁⠊⠇⠇⠑",
            "Georgia", "Eduard Shevardnadze", "საქართველო", "ედუარდ შევარდნაძე",
            "Germany", "Rudi Voeller", "Deutschland", "Rudi Völler",
            "Germany", "Walter Schultheiss", "Deutschland", "Walter Schultheiß",
            "Greece", "Giorgos Dalaras", "Ελλάς", "Γιώργος Νταλάρας",
            "Iceland", "Bjork Gudmundsdottir", "Ísland", "Björk Guðmundsdóttir",
            "India (Hindi)", "Madhuri Dixit", "भारत", "माधुरी दिछित",
            "Ireland", "Sinead O'Connor", "Éire", "Sinéad O'Connor",
            "Israel", "Yehoram Gaon", "ישראל", "יהורם גאון",
            "Italy", "Fabrizio DeAndre", "Italia", "Fabrizio De André",
            "Japan", "KUBOTA Toshinobu", "日本", "久保田    利伸",
            "Japan", "HAYASHIBARA Megumi", "日本", "林原 めぐみ",
            "Japan", "Mori Ogai", "日本", "森鷗外",
            "Japan", "Tex Texin", "日本", "テクス テクサン",
            "Norway", "Tor Age Bringsvaerd", "Noreg", "Tor Åge Bringsværd",
            "Pakistan (Urdu)", "Nusrat Fatah Ali Khan", "پاکستان", "نصرت فتح علی خان",
            "People's Rep. of China", "ZHANG Ziyi", "中国", "章子怡",
            "People's Rep. of China", "WONG Faye", "中国", "王菲",
            "Poland", "Lech Walesa", "Polska", "Lech Wałęsa",
            "Puerto Rico", "Olga Tanon", "Puerto Rico", "Olga Tañón",
            "Rep. of China", "Hsu Chi", "臺灣", "舒淇",
            "Rep. of China", "Ang Lee", "臺灣", "李安",
            "Rep. of Korea", "AHN Sung-Gi", "대한민국", "안성기",
            "Rep. of Korea", "SHIM Eun-Ha", "대한민국", "심은하",
            "Russia", "Mikhail Gorbachev", "Россия", "Михаил Горбачёв",
            "Russia", "Boris Grebenshchikov", "Россия", "Борис Гребенщиков",
            "Slovenia", "\"Frane \"\"Jezek\"\" Milcinski", "Slovenija", "Frane Milčinski - Ježek",
            "Syracuse (Sicily)", "Archimedes", "Συρακούσα", "Ἀρχιμήδης",
            "Thailand", "Thongchai McIntai", "ประเทศไทย", "ธงไชย แม็คอินไตย์",
            "U.S.A.", "Brad Pitt", "U.S.A.", "Brad Pitt",
            "Yugoslavia (Cyrillic)", "Djordje Balasevic", "Југославија", "Ђорђе Балашевић",
            "Yugoslavia (Latin)", "Djordje Balasevic", "Jugoslavija", "Đorđe Balašević"]

        i18n_arr = permutedims(reshape(i18n_data, 4, Int(floor(length(i18n_data)/4))), [2, 1])
        i18n_buff = PipeBuffer()
        writedlm(i18n_buff, i18n_arr, ',')
        @test i18n_arr == readdlm(i18n_buff, ',')

        hdr = i18n_arr[1:1, :]
        data = i18n_arr[2:end, :]
        writedlm(i18n_buff, i18n_arr, ',')
        @test (data, hdr) == readdlm(i18n_buff, ',', header=true)

        writedlm(i18n_buff, i18n_arr, '\t')
        @test (data, hdr) == readdlm(i18n_buff, '\t', header=true)
    end
end

@testset "issue #13028" begin
    for data in ["A B C", "A B C\n"]
        data,hdr = readdlm(IOBuffer(data), header=true)
        @test hdr == AbstractString["A" "B" "C"]
        @test data == Matrix{Float64}(undef, 0, 3)
    end
end

# fix #13179 parsing unicode lines with default delmiters
@test isequaldlm(readdlm(IOBuffer("# Should ignore this π\n1\tα\n2\tβ\n"), comments=true), Any[1 "α"; 2 "β"], Any)

# BigInt parser
let data = "1 2 3"
    readdlm(IOBuffer(data), ' ', BigInt) == BigInt[1 2 3]
end

@testset "show with MIME types" begin
    @test sprint(show, "text/csv", [1 2; 3 4]) == "1,2\n3,4\n"
    @test sprint(show, "text/tab-separated-values", [1 2; 3 4]) == "1\t2\n3\t4\n"

    for writefunc in ((io,x) -> show(io, "text/csv", x),
                      (io,x) -> invoke(writedlm, Tuple{IO,Any,Any}, io, x, ","))
        # iterable collections of iterable rows:
        let x = [(1,2), (3,4)], io = IOBuffer()
            writefunc(io, x)
            seek(io, 0)
            @test readdlm(io, ',') == [1 2; 3 4]
        end
        # vectors of strings:
        let x = ["foo", "bar"], io = IOBuffer()
            writefunc(io, x)
            seek(io, 0)
            @test vec(readdlm(io, ',')) == x
        end
    end

    for writefunc in ((io,x) -> show(io, "text/tab-separated-values", x),
                      (io,x) -> invoke(writedlm, Tuple{IO,Any,Any}, io, x, "\t"))
        # iterable collections of iterable rows:
        let x = [(1,2), (3,4)], io = IOBuffer()
            writefunc(io, x)
            seek(io, 0)
            @test readdlm(io, '\t') == [1 2; 3 4]
        end
        # vectors of strings:
        let x = ["foo", "bar"], io = IOBuffer()
            writefunc(io, x)
            seek(io, 0)
            @test vec(readdlm(io, '\t')) == x
        end
    end
end

# Test that we can read a write protected file
let fn = tempname()
    open(fn, "w") do f
        write(f, "Julia")
    end
    chmod(fn, 0o444)
    readdlm(fn)[] == "Julia"
    rm(fn)
end

# test writedlm with a filename instead of io input
let fn = tempname(), x = ["a" "b"; "d" ""]
    writedlm(fn, x, ',')
    @test readdlm(fn, ',') == x
    rm(fn)
end

# issue #21180
let data = "\"721\",\"1438\",\"1439\",\"…\",\"1\""
    @test readdlm(IOBuffer(data), ',') == Any[721  1438  1439  "…"  1]
end

# issue #21207
let data = "\"1\",\"灣\"\"灣灣灣灣\",\"3\""
    @test readdlm(IOBuffer(data), ',') == Any[1 "灣\"灣灣灣灣" 3]
end

# reading from a byte array (#16731)
let data = Vector{UInt8}("1,2,3\n4,5,6"), origdata = copy(data)
    @test readdlm(data, ',') == [1 2 3; 4 5 6]
    @test data == origdata
end

# issue #11484: useful error message for invalid readdlm filepath arguments
@test_throws ArgumentError readdlm(tempdir())

# displaying as text/csv
let d = TextDisplay(IOBuffer())
    display(d, "text/csv", [3 1 4])
    @test String(take!(d.io)) == "3,1,4\n"
end

@testset "complex" begin
    @test readdlm(IOBuffer("3+4im, 4+5im"), ',', Complex{Int}) == [3+4im 4+5im]
end
