Cleaning up Reddit types

Sources/SwiftNLP/1. Data Collection/Pushshift API/ReadRedditCommentFile.swift

-import Foundation
-
-func readRedditCommentFile(fileName: String) -> RedditCommentData?
-    {
-        var json: RedditCommentData?
-        if let path = Bundle.main.path(forResource: fileName, ofType: "json") {
-            do {
-                let fileUrl = URL(fileURLWithPath: path)
-                // Getting data from JSON file using the file URL
-                let data = try Data(contentsOf: fileUrl, options: .mappedIfSafe)
-                //json = try? JSONSerialization.jsonObject(with: data)
-                
-                json = try JSONDecoder().decode(RedditCommentData?.self, from: data)
-            } catch {
-                // Handle error here
-                print("Unexpected error: \(error).")
-            }
-        }
-        return json
-    }
-
-
-func readRedditCommentJson(json: Data) -> RedditCommentData? {
-    do {
-        let commentData = try JSONDecoder().decode(RedditCommentData.self, from: json)
-        return commentData
-    } catch {
-        print("Error while decoding reddit comment file: \(error)")
-        return nil
-    }
-}

Sources/SwiftNLP/1. Data Collection/Pushshift API/readRedditSubmissionData.swift

-//func readRedditSubmissionFile(fileName: String) -> RedditSubmissionData?
-//    {
-//        var json: RedditSubmissionData?
-//        if let path = Bundle.main.path(forResource: fileName, ofType: "json") {
-//            do {
-//                let fileUrl = URL(fileURLWithPath: path)
-//                // Getting data from JSON file using the file URL
-//                let data = try Data(contentsOf: fileUrl, options: .mappedIfSafe)
-//                //json = try? JSONSerialization.jsonObject(with: data)
-//
-//                json = try JSONDecoder().decode(RedditSubmissionData?.self, from: data)
-//            } catch {
-//                // Handle error here
-//                print("Unexpected error: \(error).")
-//            }
-//        }
-//        return json
-//    }
-
-import Foundation
-
-func readRedditSubmissionJson(json: Data) -> RedditSubmissionData? {
-    do {
-        let submissionData = try JSONDecoder().decode(RedditSubmissionData.self, from: json)
-        return submissionData
-    } catch {
-        print("Error while decoding reddit submission file: \(error)")
-        return nil
-    }
-}

Sources/SwiftNLP/1. Data Collection/Reddit API/Data Types/Submission.swift

@@ -45,6 +45,30 @@ public struct Submission: RedditDataItem {
     public let url: String?
     
     public var linkID: String { "\(RedditContentType.link)_\(id!)"}
+    
+    enum CodingKeys: String, CodingKey {
+        case author = "author"
+        case author_flair_css_class = "author_flair_css_class"
+        case author_flair_text = "author_flair_text"
+        case created_utc = "created_utc"
+        case domain = "domain"
+        case full_link = "full_link"
+        case id = "id"
+        case is_self = "is_self"
+        case media_embed = "media_embed"
+        case num_comments = "num_comments"
+        case over_18 = "over_18"
+        case permalink = "permalink"
+        case score = "score"
+        case hide_score = "hide_score"
+        case selftext = "selftext"
+        case subreddit = "subreddit"
+        case subreddit_id = "subreddit_id"
+        case thumbnail = "thumbnail"
+        case title = "title"
+        case url = "url"
+    }
+
 }
 
 extension Submission {

Tests/SwiftNLPTests/SNLPBoWDictionaryTests.swift

@@ -60,8 +60,8 @@ final class SwiftNLPBoWDictionaryTests: XCTestCase {
         let redditCommentJson = TestUtils.loadAllRedditComment()
         let redditSubmissionJson = TestUtils.loadAllRedditSubmission()
         
-        let redditComments = redditCommentJson.compactMap { readRedditCommentJson(json: $0) }
-        let redditSubmissions = redditSubmissionJson.compactMap { readRedditSubmissionJson(json: $0) }
+        let redditComments = redditCommentJson.compactMap { TestUtils.readRedditCommentJson(json: $0) }
+        let redditSubmissions = redditSubmissionJson.compactMap { TestUtils.readRedditSubmissionJson(json: $0) }
 
         // Extract body and selftext from each post, and store that for our corpus
         let bodies = redditComments.flatMap { $0.posts.compactMap { $0.body } } +

Tests/SwiftNLPTests/SNLPLoadDataTests.swift

@@ -13,7 +13,7 @@ final class SwiftNLPLoadDataTests: XCTestCase {
     func testRedditSubmissions() throws {
         let redditSubmissionJson = TestUtils.loadAllRedditSubmission()
         for jsonData in redditSubmissionJson {
-            let redditSubmission = readRedditSubmissionJson(json: jsonData)
+            let redditSubmission = TestUtils.readRedditSubmissionJson(json: jsonData)
             XCTAssertNotNil(redditSubmission, "Failed to decode RedditSubmissionData")
         }
     }
@@ -22,7 +22,7 @@ final class SwiftNLPLoadDataTests: XCTestCase {
     func testRedditComments() throws {
         let redditCommentJson = TestUtils.loadAllRedditComment()
         for jsonData in redditCommentJson {
-            let redditComment = readRedditCommentJson(json: jsonData)
+            let redditComment = TestUtils.readRedditCommentJson(json: jsonData)
             XCTAssertNotNil(redditComment, "Failed to decode RedditCommentData")
         }
     }
@@ -44,8 +44,8 @@ final class SwiftNLPLoadDataTests: XCTestCase {
         let redditCommentJson = TestUtils.loadAllRedditComment()
         let redditSubmissionJson = TestUtils.loadAllRedditSubmission()
         
-        let redditComments = redditCommentJson.compactMap { readRedditCommentJson(json: $0) }
-        let redditSubmissions = redditSubmissionJson.compactMap { readRedditSubmissionJson(json: $0) }
+        let redditComments = redditCommentJson.compactMap { TestUtils.readRedditCommentJson(json: $0) }
+        let redditSubmissions = redditSubmissionJson.compactMap { TestUtils.readRedditSubmissionJson(json: $0) }
 
         // Extract body and selftext from each post, and store that for our corpus
         let bodies = redditComments.flatMap { $0.posts.compactMap { $0.body } } +

Tests/SwiftNLPTests/SwiftAnnoyTest.swift

@@ -34,7 +34,7 @@ final class SwiftAnnoyTest: XCTestCase {
         
         let dictionary = corpus.dictionary
 
-        let (myIndex,myMap) = dictionary.AnnoyTest()
+        let (myIndex,_) = dictionary.AnnoyTest()
 
         var frog = dictionary["frog"]
 
@@ -44,7 +44,7 @@ final class SwiftAnnoyTest: XCTestCase {
 
         // Change the number of neighbors below
 
-        let (ids, distances) = myIndex.getNNsForVector(vector: &frog, neighbors: 10)!
+        let (_, _) = myIndex.getNNsForVector(vector: &frog, neighbors: 10)!
 
 //        for (id, distance) in zip(ids, distances) {
 //            debugPrint("\(myMap[id]!) was \(distance)")

Tests/SwiftNLPTests/TestUtil/LoadTestJson.swift

@@ -6,6 +6,7 @@
 //
 
 import Foundation
+import SwiftNLP
 
 // the goal of this util is to fetch and load json files into the Data class
 // which can then be used directly by the package to convert into objects
@@ -71,4 +72,24 @@ class TestUtils {
     static func loadAllRedditSubmission() -> [Data] {
         return loadAllFiles(prefix: "RS")
     }
+    
+    static func readRedditCommentJson(json: Data) -> RedditCommentData? {
+        do {
+            let commentData = try JSONDecoder().decode(RedditCommentData.self, from: json)
+            return commentData
+        } catch {
+            print("Error while decoding reddit comment file: \(error)")
+            return nil
+        }
+    }
+    
+    static func readRedditSubmissionJson(json: Data) -> RedditSubmissionData? {
+        do {
+            let submissionData = try JSONDecoder().decode(RedditSubmissionData.self, from: json)
+            return submissionData
+        } catch {
+            print("Error while decoding reddit submission file: \(error)")
+            return nil
+        }
+    }
 }