using PDFIO, DataFrames, Dates, CSV

# A routine to extract data from the IOM Government pdf snapshots

#folder - folder where the snapshots are stored
folder = ""

#outFold - folder where the data output csvs should be written
outFold = ""

#read in the files in the folder
lst = readdir(folder)

#select only valid files for processing
prolst = lst[lst .>= "2021-07-29.pdf"]

#initialise variables
io = IOBuffer()
testingOutput = Vector[]
frontOutput = Vector[]

#loop through each file for processing
for pdname in prolst
    
    #open pdf file for reading
    p = pdDocOpen(folder*pdname)

        #routine to extract relevant data from page 8
        pdPageExtractText(io,pdDocGetPage(p, 8))
        data = String(take!(io))

        date =
            try
                Date(match(r"Default Date\s+(\d\d/\d\d/\d\d\d\d)",data).captures[1],"dd/mm/yyyy")
            catch e
                0
            end
        tests =
            try
                parse(Int64,replace(match(r"((?:\d|,)+)\s+TESTS",data).captures[1],","=>""))
            catch e
                -1
            end
        concluded =
            try
                parse(Int64,replace(match(r"TESTS(?:.|\R)+?((?:\d|,)+)\s+TESTS",data).captures[1],","=>""))
            catch e
                -1
            end
        rate = 
            try
                parse(Float64,replace(match(r"((?:\d|\.)+)%\s+RATE",data).captures[1],"%"=>""))
            catch e
                -1
            end
        awaitresult = 
            try
                parse(Int64,replace(match(r"((?:\d|,)+)\s+AWAITING RESULT",data).captures[1],","=>""))
            catch e
                -1
            end
        bookedtest =
            try
                parse(Int64,replace(match(r"((?:\d|,)+)\s+BOOKED TESTS",data).captures[1],","=>""))
            catch e
                -1
            end
        push!(testingOutput, [date, tests, concluded, rate, awaitresult, bookedtest])

        #routine to extract relevant data from page 1
        pdPageExtractText(io,pdDocGetPage(p, 1))
        data = String(take!(io))

        date =
        try
            Date(match(r"Data Last Refreshed\s+(\d\d/\d\d/\d\d\d\d)",data).captures[1],"dd/mm/yyyy")
        catch e
            0
        end

        topush = String[]
        for m in collect(eachmatch(r"(?:\w|\s)((?:\d|,)+)(?:\w|\s)",data))
            push!(topush, m.captures[1])
        end
        for n in length(topush):20
            push!(topush,"-1")
        end
        topush
        
        push!(frontOutput, [date, topush...])

    pdDocClose(p)
    
end


CSV.write(outFold*"front.csv",DataFrame(frontOutput))
CSV.write(outFold*"testing.csv",DataFrame(testingOutput))