Added basic function to fetch all comments from a submission, returning a thread.

Sources/SwiftNLP/1. Data Collection/Reddit API/Data Types/MoreComments.swift

@@ -33,3 +33,16 @@ struct RessidtListingMore: RedditDataItem {
     var id: String? { return nil }
     var created_utc: Int32? { return nil } //  TODO: This is a hack that allows conformance to RedditDataItem ... fix later?
 }
+
+struct MoreContainer: Codable {
+    var json: InnerMoreContainer
+}
+
+struct InnerMoreContainer: Codable {
+    let errors: [String]
+    let data: InnerInnerMoreContainer
+}
+
+struct InnerInnerMoreContainer: Codable {
+    let things: [ListingDataItem]
+}

Sources/SwiftNLP/1. Data Collection/Reddit API/Data Types/Submission.swift

@@ -43,6 +43,8 @@ public struct Submission: RedditDataItem {
     public let thumbnail: String?
     public let title: String?
     public let url: String?
+    
+    public var linkID: String { "\(RedditContentType.link)_\(id!)"}
 }
 
 extension Submission {

Sources/SwiftNLP/1. Data Collection/Reddit API/Network Endpoints/Session + Comment Search.swift

@@ -27,7 +27,9 @@ extension Session {
     
     /// Returns a comment tree corresponding to a search of the r/subreddit/comments/article endpoint
     func searchComment(
-        submission: Submission,
+        subreddit: String,
+        articleID: String,
+        //submission: Submission,
         comment: String? = nil,
         context: UInt = 0,
         depth: Int? = nil,
@@ -43,10 +45,10 @@ extension Session {
         truncate: UInt = 0        
     ) async throws -> [Listing] {
                
-        guard let subreddit = submission.subreddit, let articleID = submission.id
-        else {
-            throw SessionError(message: "Submission must include article data.")
-        }
+//        guard let subreddit = submission.subreddit, let articleID = submission.id
+//        else {
+//            throw SessionError(message: "Submission must include article data.")
+//        }
         
         var parameters: [String : String] = [String:String]()
         

Sources/SwiftNLP/1. Data Collection/Reddit API/Network Endpoints/Session + Info.swift

+// Copyright (c) 2023 Jim Wallace
+//
+// Permission is hereby granted, free of charge, to any person
+// obtaining a copy of this software and associated documentation
+// files (the "Software"), to deal in the Software without
+// restriction, including without limitation the rights to use,
+// copy, modify, merge, publish, distribute, sublicense, and/or sell
+// copies of the Software, and to permit persons to whom the
+// Software is furnished to do so, subject to the following
+// conditions:
+//
+// The above copyright notice and this permission notice shall be
+// included in all copies or substantial portions of the Software.
+//
+// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+// EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
+// OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+// NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
+// HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
+// WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+// OTHER DEALINGS IN THE SOFTWARE.
+
+import Foundation
+
+extension Session {
+ 
+    // TODO: Add caller for the /api/info endpoint
+    // This should give information about an arbitrary data point, useful, but is an additional call in many cases and so should be avoided
+    
+}

Sources/SwiftNLP/1. Data Collection/Reddit API/Network Endpoints/Session + MoreChildren.swift

@@ -29,11 +29,11 @@ extension Session {
     func moreChildren(
         linkID: String,
         children: [String],
-        id: String?,
+        id: String? = nil,
         depth: Int? = nil,
         limitChildren: Bool = false,
         sort: ListingSortOrder = .new
-    ) async throws -> [Listing] {
+    ) async throws -> ([Comment],[String]) {
                
         
         var parameters: [String : String] = [String:String]()
@@ -59,10 +59,29 @@ extension Session {
         )
         
         do {
-            return try JSONDecoder().decode([Listing].self, from: data)
+            let container = try JSONDecoder().decode(MoreContainer.self, from: data)
+            return _processContainer(container)
+            
         } catch {
             throw SessionError(message: "Unable to decode server response.")
         }
     }
     
+    func _processContainer(_ c: MoreContainer) -> ([Comment], [String]) {
+        var comments: [Comment] = [Comment]()
+        var more: [String] = [String]()
+        
+        for child in c.json.data.things {
+            if child.kind == .comment {
+                comments.append(child.data as! Comment)
+            }
+            if child.kind == .more {
+                let moreItems = child.data as! RessidtListingMore
+                more.append(contentsOf: moreItems.children)
+            }
+        }
+        return (comments,more)
+    }
+    
+
 }

Sources/SwiftNLP/1. Data Collection/Reddit API/Session.swift

@@ -63,13 +63,57 @@ class Session {
 
 }
 
-//extension Session {
-//    
-//    func fetchThread(submissionID: String) -> RedditThread? {
-//        
-//        
-//        
-//        return nil
-//    }
-//    
-//}
+extension Session {
+    
+    func fetchThread(subreddit: String, articleID: String) async throws -> RedditThread {
+        
+        // TODO: What are the optimal parameters for this call?
+        let listings = try await searchComment(subreddit: subreddit, articleID: articleID, showMore: true)
+        
+        // First listing should contain our submission data
+        guard listings[0].data.children[0].kind == .link else {
+            throw SessionError(message: "Error, did not find initial post for this subreddit and articleID")
+        }
+        let submission = listings[0].data.children[0].data as! Submission
+        
+        // Iteratively unwrap the MoreComments results, until we're all done
+        var (comments, more) = _processListingIntoCommentsAndMore(listings[1])
+        
+        // While we have more comments to fetch, keep making calls to the MoreChildren endpoint
+        while !more.isEmpty {
+            
+            var toFetch: [String] = [String]()
+            
+            if more.count > 100 {
+                toFetch = Array(more[0 ..< 100])
+                more = Array(more[100 ..< more.count ])
+            } else {
+                toFetch = more
+                more = [String]()
+            }
+            
+            let (c2, m2) = try await moreChildren(linkID: submission.linkID, children: toFetch)
+            
+            // Add comments to our list of comments, add more items to our list of more items
+            comments.append(contentsOf: c2)
+            more.append(contentsOf: m2)
+        }
+        return RedditThread(submission: submission, comments: comments)
+    }
+    
+    func _processListingIntoCommentsAndMore(_ l: Listing) -> ([Comment], [String]) {
+        var comments: [Comment] = [Comment]()
+        var more: [String] = [String]()
+        for child in l.data.children {
+            if child.kind == .comment {
+                comments.append(child.data as! Comment)
+            }
+            if child.kind == .more {
+                let moreItems = child.data as! RessidtListingMore
+                more.append(contentsOf: moreItems.children)
+            }
+        }
+        return (comments,more)
+    }
+    
+}

Tests/SwiftNLPTests/Reddit API/Session Method Tests.swift

+// Copyright (c) 2023 Jim Wallace
+//
+// Permission is hereby granted, free of charge, to any person
+// obtaining a copy of this software and associated documentation
+// files (the "Software"), to deal in the Software without
+// restriction, including without limitation the rights to use,
+// copy, modify, merge, publish, distribute, sublicense, and/or sell
+// copies of the Software, and to permit persons to whom the
+// Software is furnished to do so, subject to the following
+// conditions:
+//
+// The above copyright notice and this permission notice shall be
+// included in all copies or substantial portions of the Software.
+//
+// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+// EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
+// OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+// NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
+// HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
+// WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+// OTHER DEALINGS IN THE SOFTWARE.
+
+import Foundation
+import XCTest
+@testable import SwiftNLP
+
+final class RedditSessionMethodTest: XCTestCase {
+    
+    
+    func testFetchComments() async throws {
+        let id = ProcessInfo.processInfo.environment["REDDIT_CLIENT_ID"] ?? nil
+        let secret = ProcessInfo.processInfo.environment["REDDIT_CLIENT_SECRET"] ?? nil
+        
+        guard let id = id, let secret = secret else {
+            fatalError("Unable to fetch REDDIT_CLIENT_ID and REDDIT_CLIENT_SECRET from ProcessInfo.")
+        }
+        
+        let client = Session(id: id, secret: secret)
+        guard let _ = try? await client.authenticate() else {
+            throw SessionError(message: "Error authenticating client.")
+        }
+        
+        // "7dljcy", subreddit: "AskReddit")
+        // https://www.reddit.com/r/redditdev/comments/7dohn2/why_does_the_following_api_endpoint_return/
+        let result: RedditThread = try await client.fetchThread(subreddit: "AskReddit", articleID: "7dljcy") // TODO: We aren't loading all 2610 comments... some deleted, but can we tune the method to get more? 
+        //print("Loaded thread with \(result.comments.count) comments")
+        //print(result.submission)
+        //print(result.comments[100])
+        
+        XCTAssert(result.submission.num_comments == 2610 && result.submission.subreddit == "AskReddit")
+    }
+}

Tests/SwiftNLPTests/Reddit API/Session Tests.swift → Tests/SwiftNLPTests/Reddit API/Session Network Endpoint Tests.swift

+// Copyright (c) 2023 Jim Wallace
 //
-//  File.swift
+// Permission is hereby granted, free of charge, to any person
+// obtaining a copy of this software and associated documentation
+// files (the "Software"), to deal in the Software without
+// restriction, including without limitation the rights to use,
+// copy, modify, merge, publish, distribute, sublicense, and/or sell
+// copies of the Software, and to permit persons to whom the
+// Software is furnished to do so, subject to the following
+// conditions:
 //
+// The above copyright notice and this permission notice shall be
+// included in all copies or substantial portions of the Software.
 //
-//  Created by Nicole Mathis on 2023-07-04.
-//
-
-
-// When increasing the number of trees, the output is significantly more accurate however, the time to run becomes much longer
+// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+// EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
+// OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+// NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
+// HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
+// WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+// OTHER DEALINGS IN THE SOFTWARE.
 
 import Foundation
 import XCTest
 @testable import SwiftNLP
 
-final class RedditSessionTest: XCTestCase {
+final class RedditSessionEndpointTest: XCTestCase {
     
     
     func testHasRedditCredentials() throws {
@@ -125,10 +138,10 @@ final class RedditSessionTest: XCTestCase {
         }
         
         // https://www.reddit.com/r/uwaterloo/comments/18lbokl/conestoga_college_finally_being_called_out_by_the/
-        let submission = Submission(id: "18lbokl", subreddit: "uwaterloo")
+        //let submission = Submission(id: "18lbokl", subreddit: "uwaterloo")
         
         // This should return an array of listings, one with original submisison and one with responses.
-        let result = try await client.searchComment(submission: submission)
+        let result = try await client.searchComment(subreddit: "uwaterloo", articleID: "18lbokl")
         
         
         XCTAssert(result.count > 0)
@@ -151,20 +164,20 @@ final class RedditSessionTest: XCTestCase {
         
         // 1) https://www.reddit.com/r/AmItheAsshole/comments/18m3xgr/aita_for_refusing_to_attend_my_inlaws_christmas/
         // 2) https://www.reddit.com/r/AskReddit/comments/7dljcy/serious_what_can_the_average_joe_do_to_save_net/.json
-        let submission = Submission(id: "7dljcy", subreddit: "AskReddit")
+        //let submission = Submission(id: "7dljcy", subreddit: "AskReddit")
         
         // This should return an array of listings, one with original submisison and one with responses.
         // It's a big thread, so we *should* also get a `more` entry
-        let result = try await client.searchComment(submission: submission, showMore: true)
-        
-        for (i, listing) in result.enumerated() {
-            print("Listing \(i): \(listing.data.children.count) elements")
-            for item in listing.data.children {
-                if item.kind == .more {
-                    print(item)
-                }
-            }
-        }
+        let result = try await client.searchComment(subreddit: "AskReddit", articleID: "7dljcy", showMore: true)
+        
+//        for (i, listing) in result.enumerated() {
+//            print("Listing \(i): \(listing.data.children.count) elements")
+//            for item in listing.data.children {
+//                if item.kind == .more {
+//                    print(item)
+//                }
+//            }
+//        }
                 
         XCTAssert(result.count > 0)
         //XCTAssert(result[1].children.count > 0)