diff --git a/build-locate-id.sh b/Locate ID/build-locate-id.sh similarity index 77% rename from build-locate-id.sh rename to Locate ID/build-locate-id.sh index 0c3ad79..a22405f 100755 --- a/build-locate-id.sh +++ b/Locate ID/build-locate-id.sh @@ -1,8 +1,9 @@ #!/usr/bin/env bash + go mod download -package="./Locate ID/LocateID.go" +package="./LocateID.go" package_split=(${package//\// }) package_name="LocateID" @@ -15,11 +16,11 @@ do GOOS=${platform_split[0]} GOARCH=${platform_split[1]} output_name=$package_name'-'$GOOS'-'$GOARCH - if [ $GOOS = "windows" ]; then + if [ "$GOOS" = "windows" ]; then output_name+='.exe' fi - env GOOS=$GOOS GOARCH=$GOARCH go build -o "./Locate ID/build/$output_name" $package + env GOOS="$GOOS" GOARCH="$GOARCH" go build -o ./build/$output_name $package if [ $? -ne 0 ]; then echo 'An error has occurred! Aborting the script execution...' exit 1 diff --git a/README.md b/README.md index c096e96..efd1514 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,4 @@ -# Discord Chat Exporter csv parsers +# [Discord Chat Exporter](https://github.com/Tyrrrz/DiscordChatExporter) csv parsers Now with a much faster version written in go! @@ -9,7 +9,7 @@ To run the go versions all you need to do is download the executable for your platform and run it from the terminal/powershell: ```bash -./IdParser[.exe] -i [Path to folder/file] -o [Path to output .csv (Defaults to ./out.csv)] +./ParseIDs[.exe] -i [Path to folder/file] -o [Path to output .csv (Defaults to ./out.csv)] ``` Use the same output between multiple runs works and takes into account old data to ensure no dupes diff --git a/User ID Parser/IdParser.go b/User ID Parser/IdParser.go index d4d9570..7ced6f7 100644 --- a/User ID Parser/IdParser.go +++ b/User ID Parser/IdParser.go @@ -1,21 +1,25 @@ package main import ( + "bytes" "encoding/csv" "errors" "flag" "fmt" + "io" "io/ioutil" "log" "os" + "strings" ) // Variables used for command line parameters var ( - CsvInput string - CsvOutput string - unique_old []string - unique_ids []string + CsvInput string + CsvOutput string + unique_old []string + unique_name []string + unique_ids []string ) func init() { @@ -27,7 +31,8 @@ func init() { // struct for getting just user ID out of csv file type DiscordCSV struct { - UserID string + UserID string + Username string } func main() { @@ -56,9 +61,17 @@ func main() { // For each file in folder for _, file := range files { if !file.IsDir() { - process(CsvInput + file.Name()) - } + // file is not a directory so process it's full path + // check if windows or linux + if strings.Contains(CsvInput, "\\") { + // windows + process(CsvInput + "\\" + file.Name()) + } else { + // linux + process(CsvInput + "/" + file.Name()) + } + } } } else { @@ -77,16 +90,21 @@ func process(CsvPath string) { // Read CSV file lines, err := ReadCsv(CsvOutput) if err != nil { - panic(err) - } - // Loop through lines & turn into object - for _, line := range lines { - data := DiscordCSV{ - UserID: line[0], - } - unique_old = append(unique_old, data.UserID) + return } + // Loop through lines & add to DiscordIDs list + for i, _ := range lines { + + data := DiscordCSV{ + UserID: lines[i][0], + Username: lines[i][1], + } + if !contains(unique_old, data.UserID) { + unique_old = append(unique_old, data.UserID) + unique_name = append(unique_name, data.Username) + } + } WriteCSV(CsvPath) } else if errors.Is(err, os.ErrNotExist) { @@ -112,6 +130,7 @@ func process(CsvPath string) { func WriteCSV(InFile string) { // Create DiscordIDs list var DiscordIDs []string + var DiscordNames []string // Read CSV file lines, err := ReadCsv(InFile) @@ -122,20 +141,23 @@ func WriteCSV(InFile string) { OutFile, err := os.OpenFile(CsvOutput, os.O_APPEND|os.O_WRONLY|os.O_CREATE, 0600) // Loop through lines & add to DiscordIDs list - for _, line := range lines { + for i, _ := range lines { + data := DiscordCSV{ - UserID: line[0], + UserID: lines[i][0], + Username: lines[i][1], } if !contains(DiscordIDs, data.UserID) { DiscordIDs = append(DiscordIDs, data.UserID) + DiscordNames = append(DiscordNames, data.Username) } } // Checks if ID is already in list // if it isnt then it gets writen to list - for _, ID := range DiscordIDs { + for i, ID := range DiscordIDs { if !contains(unique_old, ID) { - OutFile.WriteString(ID + "\n") + OutFile.WriteString(ID + "," + DiscordNames[i] + "\n") } } } @@ -149,8 +171,21 @@ func ReadCsv(filename string) ([][]string, error) { } defer f.Close() + var buf bytes.Buffer + io.Copy(&buf, f) + + x := string(buf.Bytes()) + + strings.ReplaceAll(x, "\r\n", "\n") + + strings.ReplaceAll(x, "\r\n", " ") + + reader := csv.NewReader(strings.NewReader(x)) + + reader.ReuseRecord = true + // Read File into a Variable - lines, err := csv.NewReader(f).ReadAll() + lines, err := reader.ReadAll() if err != nil { return [][]string{}, err } diff --git a/build-id-parser.sh b/User ID Parser/build-id-parser.sh similarity index 76% rename from build-id-parser.sh rename to User ID Parser/build-id-parser.sh index 79011a2..18fb6b4 100755 --- a/build-id-parser.sh +++ b/User ID Parser/build-id-parser.sh @@ -2,7 +2,7 @@ go mod download -package="./User ID Praser/IdParser.go" +package="./IdParser.go" package_split=(${package//\// }) package_name="IdParser" @@ -15,11 +15,11 @@ do GOOS=${platform_split[0]} GOARCH=${platform_split[1]} output_name=$package_name'-'$GOOS'-'$GOARCH - if [ $GOOS = "windows" ]; then + if [ "$GOOS" = "windows" ]; then output_name+='.exe' fi - env GOOS=$GOOS GOARCH=$GOARCH go build -o ./User ID Praser/build/$output_name $package + env GOOS="$GOOS" GOARCH=$GOARCH go build -o "./build/$output_name" $package if [ $? -ne 0 ]; then echo 'An error has occurred! Aborting the script execution...' exit 1 diff --git a/requirements.txt b/requirements.txt index f8804f8..b9ffc02 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1 +1,2 @@ pandas~=1.3.1 +numpy>=1.22.2 # not directly required, pinned by Snyk to avoid a vulnerability