From 58c624300f5955f5f75a320a9d63def7c6ef1756 Mon Sep 17 00:00:00 2001 From: Owen Quinlan <46948241+BuyMyMojo@users.noreply.github.com> Date: Fri, 19 Nov 2021 21:45:04 +1100 Subject: [PATCH 1/9] change build scripts to work with TeamCity --- build-id-parser.sh | 4 ++-- build-locate-id.sh | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/build-id-parser.sh b/build-id-parser.sh index 79011a2..b9763d8 100755 --- a/build-id-parser.sh +++ b/build-id-parser.sh @@ -2,7 +2,7 @@ go mod download -package="./User ID Praser/IdParser.go" +package="./User ID Parser/IdParser.go" package_split=(${package//\// }) package_name="IdParser" @@ -19,7 +19,7 @@ do output_name+='.exe' fi - env GOOS=$GOOS GOARCH=$GOARCH go build -o ./User ID Praser/build/$output_name $package + env GOOS=$GOOS GOARCH=$GOARCH GOPATH="./User ID Parser/" go build -o ./User ID Praser/build/$output_name $package if [ $? -ne 0 ]; then echo 'An error has occurred! Aborting the script execution...' exit 1 diff --git a/build-locate-id.sh b/build-locate-id.sh index 0c3ad79..89619bc 100755 --- a/build-locate-id.sh +++ b/build-locate-id.sh @@ -19,7 +19,7 @@ do output_name+='.exe' fi - env GOOS=$GOOS GOARCH=$GOARCH go build -o "./Locate ID/build/$output_name" $package + env GOOS=$GOOS GOARCH=$GOARCH GOPATH="./Locate ID/" go build -o "./Locate ID/build/$output_name" $package if [ $? -ne 0 ]; then echo 'An error has occurred! Aborting the script execution...' exit 1 From 69023df4286fa108a6942325ba0f996342c1fd12 Mon Sep 17 00:00:00 2001 From: Owen Quinlan <46948241+BuyMyMojo@users.noreply.github.com> Date: Fri, 19 Nov 2021 21:48:07 +1100 Subject: [PATCH 2/9] change build scripts to work with TeamCity --- build-id-parser.sh | 4 ++-- build-locate-id.sh | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/build-id-parser.sh b/build-id-parser.sh index b9763d8..8f67a0f 100755 --- a/build-id-parser.sh +++ b/build-id-parser.sh @@ -2,7 +2,7 @@ go mod download -package="./User ID Parser/IdParser.go" +package="$PWD/User ID Parser/IdParser.go" package_split=(${package//\// }) package_name="IdParser" @@ -19,7 +19,7 @@ do output_name+='.exe' fi - env GOOS=$GOOS GOARCH=$GOARCH GOPATH="./User ID Parser/" go build -o ./User ID Praser/build/$output_name $package + env GOOS=$GOOS GOARCH=$GOARCH GOPATH="$PWD/User ID Parser/" go build -o ./User ID Praser/build/$output_name $package if [ $? -ne 0 ]; then echo 'An error has occurred! Aborting the script execution...' exit 1 diff --git a/build-locate-id.sh b/build-locate-id.sh index 89619bc..6460d2e 100755 --- a/build-locate-id.sh +++ b/build-locate-id.sh @@ -2,7 +2,7 @@ go mod download -package="./Locate ID/LocateID.go" +package="$PWD/Locate ID/LocateID.go" package_split=(${package//\// }) package_name="LocateID" @@ -19,7 +19,7 @@ do output_name+='.exe' fi - env GOOS=$GOOS GOARCH=$GOARCH GOPATH="./Locate ID/" go build -o "./Locate ID/build/$output_name" $package + env GOOS=$GOOS GOARCH=$GOARCH GOPATH="$PWD/Locate ID/" go build -o "./Locate ID/build/$output_name" $package if [ $? -ne 0 ]; then echo 'An error has occurred! Aborting the script execution...' exit 1 From 55a1d488245add8f198727c54bb28ce7e72512d0 Mon Sep 17 00:00:00 2001 From: Owen Quinlan <46948241+BuyMyMojo@users.noreply.github.com> Date: Fri, 19 Nov 2021 22:09:36 +1100 Subject: [PATCH 3/9] change build scripts to work with TeamCity --- build-locate-id.sh => Locate ID/build-locate-id.sh | 7 ++++--- build-id-parser.sh => User ID Parser/build-id-parser.sh | 6 +++--- 2 files changed, 7 insertions(+), 6 deletions(-) rename build-locate-id.sh => Locate ID/build-locate-id.sh (74%) rename build-id-parser.sh => User ID Parser/build-id-parser.sh (73%) diff --git a/build-locate-id.sh b/Locate ID/build-locate-id.sh similarity index 74% rename from build-locate-id.sh rename to Locate ID/build-locate-id.sh index 6460d2e..a22405f 100755 --- a/build-locate-id.sh +++ b/Locate ID/build-locate-id.sh @@ -1,8 +1,9 @@ #!/usr/bin/env bash + go mod download -package="$PWD/Locate ID/LocateID.go" +package="./LocateID.go" package_split=(${package//\// }) package_name="LocateID" @@ -15,11 +16,11 @@ do GOOS=${platform_split[0]} GOARCH=${platform_split[1]} output_name=$package_name'-'$GOOS'-'$GOARCH - if [ $GOOS = "windows" ]; then + if [ "$GOOS" = "windows" ]; then output_name+='.exe' fi - env GOOS=$GOOS GOARCH=$GOARCH GOPATH="$PWD/Locate ID/" go build -o "./Locate ID/build/$output_name" $package + env GOOS="$GOOS" GOARCH="$GOARCH" go build -o ./build/$output_name $package if [ $? -ne 0 ]; then echo 'An error has occurred! Aborting the script execution...' exit 1 diff --git a/build-id-parser.sh b/User ID Parser/build-id-parser.sh similarity index 73% rename from build-id-parser.sh rename to User ID Parser/build-id-parser.sh index 8f67a0f..18fb6b4 100755 --- a/build-id-parser.sh +++ b/User ID Parser/build-id-parser.sh @@ -2,7 +2,7 @@ go mod download -package="$PWD/User ID Parser/IdParser.go" +package="./IdParser.go" package_split=(${package//\// }) package_name="IdParser" @@ -15,11 +15,11 @@ do GOOS=${platform_split[0]} GOARCH=${platform_split[1]} output_name=$package_name'-'$GOOS'-'$GOARCH - if [ $GOOS = "windows" ]; then + if [ "$GOOS" = "windows" ]; then output_name+='.exe' fi - env GOOS=$GOOS GOARCH=$GOARCH GOPATH="$PWD/User ID Parser/" go build -o ./User ID Praser/build/$output_name $package + env GOOS="$GOOS" GOARCH=$GOARCH go build -o "./build/$output_name" $package if [ $? -ne 0 ]; then echo 'An error has occurred! Aborting the script execution...' exit 1 From 5547316ff302c04652fcfec8dc0ed6136653c954 Mon Sep 17 00:00:00 2001 From: Owen Quinlan <46948241+BuyMyMojo@users.noreply.github.com> Date: Fri, 19 Nov 2021 23:18:31 +1100 Subject: [PATCH 4/9] change build scripts to work with TeamCity --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index c096e96..97e8547 100644 --- a/README.md +++ b/README.md @@ -9,7 +9,7 @@ To run the go versions all you need to do is download the executable for your platform and run it from the terminal/powershell: ```bash -./IdParser[.exe] -i [Path to folder/file] -o [Path to output .csv (Defaults to ./out.csv)] +./ParseIDs[.exe] -i [Path to folder/file] -o [Path to output .csv (Defaults to ./out.csv)] ``` Use the same output between multiple runs works and takes into account old data to ensure no dupes From f627e97bb1863e433d41995b02528022a85ffd31 Mon Sep 17 00:00:00 2001 From: BuyMyMojo Date: Thu, 21 Jul 2022 00:23:43 +1000 Subject: [PATCH 5/9] feat: :sparkles: Log usernames with the IDs --- User ID Parser/IdParser.go | 46 +++++++++++++++++++++++++------------- 1 file changed, 30 insertions(+), 16 deletions(-) diff --git a/User ID Parser/IdParser.go b/User ID Parser/IdParser.go index d4d9570..114047f 100644 --- a/User ID Parser/IdParser.go +++ b/User ID Parser/IdParser.go @@ -12,10 +12,11 @@ import ( // Variables used for command line parameters var ( - CsvInput string - CsvOutput string - unique_old []string - unique_ids []string + CsvInput string + CsvOutput string + unique_old []string + unique_name []string + unique_ids []string ) func init() { @@ -27,7 +28,8 @@ func init() { // struct for getting just user ID out of csv file type DiscordCSV struct { - UserID string + UserID string + Username string } func main() { @@ -79,14 +81,19 @@ func process(CsvPath string) { if err != nil { panic(err) } - // Loop through lines & turn into object - for _, line := range lines { - data := DiscordCSV{ - UserID: line[0], - } - unique_old = append(unique_old, data.UserID) - } + // Loop through lines & add to DiscordIDs list + for i, _ := range lines { + + data := DiscordCSV{ + UserID: lines[i][0], + Username: lines[i][1], + } + if !contains(unique_old, data.UserID) { + unique_old = append(unique_old, data.UserID) + unique_name = append(unique_name, data.Username) + } + } WriteCSV(CsvPath) } else if errors.Is(err, os.ErrNotExist) { @@ -112,6 +119,7 @@ func process(CsvPath string) { func WriteCSV(InFile string) { // Create DiscordIDs list var DiscordIDs []string + var DiscordNames []string // Read CSV file lines, err := ReadCsv(InFile) @@ -122,20 +130,23 @@ func WriteCSV(InFile string) { OutFile, err := os.OpenFile(CsvOutput, os.O_APPEND|os.O_WRONLY|os.O_CREATE, 0600) // Loop through lines & add to DiscordIDs list - for _, line := range lines { + for i, _ := range lines { + data := DiscordCSV{ - UserID: line[0], + UserID: lines[i][0], + Username: lines[i][1], } if !contains(DiscordIDs, data.UserID) { DiscordIDs = append(DiscordIDs, data.UserID) + DiscordNames = append(DiscordNames, data.Username) } } // Checks if ID is already in list // if it isnt then it gets writen to list - for _, ID := range DiscordIDs { + for i, ID := range DiscordIDs { if !contains(unique_old, ID) { - OutFile.WriteString(ID + "\n") + OutFile.WriteString(ID + "," + DiscordNames[i] + "\n") } } } @@ -149,6 +160,9 @@ func ReadCsv(filename string) ([][]string, error) { } defer f.Close() + var x string + f.WriteString(x) + // Read File into a Variable lines, err := csv.NewReader(f).ReadAll() if err != nil { From b3f48957b109a22904bfdc3b3e9d76508cdcf594 Mon Sep 17 00:00:00 2001 From: BuyMyMojo Date: Thu, 21 Jul 2022 01:32:11 +1000 Subject: [PATCH 6/9] fix: :ambulance: Ignore useless error --- User ID Parser/IdParser.go | 21 +++++++++++++++++---- 1 file changed, 17 insertions(+), 4 deletions(-) diff --git a/User ID Parser/IdParser.go b/User ID Parser/IdParser.go index 114047f..912ae3c 100644 --- a/User ID Parser/IdParser.go +++ b/User ID Parser/IdParser.go @@ -1,13 +1,16 @@ package main import ( + "bytes" "encoding/csv" "errors" "flag" "fmt" + "io" "io/ioutil" "log" "os" + "strings" ) // Variables used for command line parameters @@ -79,7 +82,7 @@ func process(CsvPath string) { // Read CSV file lines, err := ReadCsv(CsvOutput) if err != nil { - panic(err) + return } // Loop through lines & add to DiscordIDs list @@ -160,11 +163,21 @@ func ReadCsv(filename string) ([][]string, error) { } defer f.Close() - var x string - f.WriteString(x) + var buf bytes.Buffer + io.Copy(&buf, f) + + x := string(buf.Bytes()) + + strings.ReplaceAll(x, "\r\n", "\n") + + strings.ReplaceAll(x, "\r\n", " ") + + reader := csv.NewReader(strings.NewReader(x)) + + reader.ReuseRecord = true // Read File into a Variable - lines, err := csv.NewReader(f).ReadAll() + lines, err := reader.ReadAll() if err != nil { return [][]string{}, err } From a28999f92f215bdf9e90d62bb82d00de6f44d223 Mon Sep 17 00:00:00 2001 From: BuyMyMojo Date: Wed, 21 Sep 2022 23:39:37 +1000 Subject: [PATCH 7/9] Change input path handling Update input path to fix weird windows issues for a user. --- User ID Parser/IdParser.go | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/User ID Parser/IdParser.go b/User ID Parser/IdParser.go index 912ae3c..7ced6f7 100644 --- a/User ID Parser/IdParser.go +++ b/User ID Parser/IdParser.go @@ -61,9 +61,17 @@ func main() { // For each file in folder for _, file := range files { if !file.IsDir() { - process(CsvInput + file.Name()) - } + // file is not a directory so process it's full path + // check if windows or linux + if strings.Contains(CsvInput, "\\") { + // windows + process(CsvInput + "\\" + file.Name()) + } else { + // linux + process(CsvInput + "/" + file.Name()) + } + } } } else { From 30e6dcefff7c81d8b82d70db50e10d4bbbf144b5 Mon Sep 17 00:00:00 2001 From: Owen Quinlan Date: Mon, 26 Sep 2022 13:04:50 +1000 Subject: [PATCH 8/9] Update README.md --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 97e8547..efd1514 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,4 @@ -# Discord Chat Exporter csv parsers +# [Discord Chat Exporter](https://github.com/Tyrrrz/DiscordChatExporter) csv parsers Now with a much faster version written in go! From 2d0179bb9ed40d907f0496ca611569d05477a719 Mon Sep 17 00:00:00 2001 From: snyk-bot Date: Thu, 16 Feb 2023 20:21:38 +0000 Subject: [PATCH 9/9] fix: requirements.txt to reduce vulnerabilities The following vulnerabilities are fixed by pinning transitive dependencies: - https://snyk.io/vuln/SNYK-PYTHON-NUMPY-2321964 - https://snyk.io/vuln/SNYK-PYTHON-NUMPY-2321966 - https://snyk.io/vuln/SNYK-PYTHON-NUMPY-2321970 --- requirements.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/requirements.txt b/requirements.txt index f8804f8..b9ffc02 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1 +1,2 @@ pandas~=1.3.1 +numpy>=1.22.2 # not directly required, pinned by Snyk to avoid a vulnerability